diff options
author | iaz1607 <iaz1607@yandex-team.ru> | 2022-02-10 16:45:37 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:37 +0300 |
commit | e5437feb4ac2d2dc044e1090b9312dde5ef197e0 (patch) | |
tree | f5a238c69dd20a1fa2092127a31b8aff25020f7d /library/python | |
parent | f4945d0a44b8770f0801de3056aa41639b0b7bd2 (diff) | |
download | ydb-e5437feb4ac2d2dc044e1090b9312dde5ef197e0.tar.gz |
Restoring authorship annotation for <iaz1607@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'library/python')
19 files changed, 372 insertions, 372 deletions
diff --git a/library/python/fs/__init__.py b/library/python/fs/__init__.py index b1b7cde079..da4bd711da 100644 --- a/library/python/fs/__init__.py +++ b/library/python/fs/__init__.py @@ -326,8 +326,8 @@ def write_file(path, data, binary=True): ensure_dir(dir_path) tmp_path = path + '.tmp.' + str(random.random()) with open_file(tmp_path, 'w' + ('b' if binary else '')) as f: - if not isinstance(data, bytes) and binary: - data = data.encode('UTF-8') + if not isinstance(data, bytes) and binary: + data = data.encode('UTF-8') f.write(data) replace_file(tmp_path, path) diff --git a/library/python/pytest/context.py b/library/python/pytest/context.py index bfcdae50b5..be1b8854ae 100644 --- a/library/python/pytest/context.py +++ b/library/python/pytest/context.py @@ -1 +1 @@ -Ctx = {} +Ctx = {} diff --git a/library/python/pytest/main.py b/library/python/pytest/main.py index 6296bd6f0f..c9e0cdf48e 100644 --- a/library/python/pytest/main.py +++ b/library/python/pytest/main.py @@ -1,6 +1,6 @@ import os import sys -import time +import time import __res @@ -8,9 +8,9 @@ FORCE_EXIT_TESTSFAILED_ENV = 'FORCE_EXIT_TESTSFAILED' def main(): - import library.python.pytest.context as context - context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time() - + import library.python.pytest.context as context + context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time() + profile = None if '--profile-pytest' in sys.argv: sys.argv.remove('--profile-pytest') diff --git a/library/python/pytest/plugins/collection.py b/library/python/pytest/plugins/collection.py index e36f47a78f..86e0d68a9b 100644 --- a/library/python/pytest/plugins/collection.py +++ b/library/python/pytest/plugins/collection.py @@ -7,8 +7,8 @@ import py import pytest # noqa import _pytest.python import _pytest.doctest -import json -import library.python.testing.filter.filter as test_filter +import json +import library.python.testing.filter.filter as test_filter class LoadedModule(_pytest.python.Module): @@ -78,13 +78,13 @@ class DoctestModule(LoadedModule): # NOTE: Since we are overriding collect method of pytest session, pytest hooks are not invoked during collection. -def pytest_ignore_collect(module, session, filenames_from_full_filters, accept_filename_predicate): - if session.config.option.mode == 'list': - return not accept_filename_predicate(module.name) - - if filenames_from_full_filters is not None and module.name not in filenames_from_full_filters: - return True - +def pytest_ignore_collect(module, session, filenames_from_full_filters, accept_filename_predicate): + if session.config.option.mode == 'list': + return not accept_filename_predicate(module.name) + + if filenames_from_full_filters is not None and module.name not in filenames_from_full_filters: + return True + test_file_filter = getattr(session.config.option, 'test_file_filter', None) if test_file_filter is None: return False @@ -101,21 +101,21 @@ class CollectionPlugin(object): def pytest_sessionstart(self, session): def collect(*args, **kwargs): - accept_filename_predicate = test_filter.make_py_file_filter(session.config.option.test_filter) - full_test_names_file_path = session.config.option.test_list_path - filenames_filter = None - - if full_test_names_file_path and os.path.exists(full_test_names_file_path): - with open(full_test_names_file_path, 'r') as afile: - # in afile stored 2 dimensional array such that array[modulo_index] contains tests which should be run in this test suite - full_names_filter = set(json.load(afile)[int(session.config.option.modulo_index)]) - filenames_filter = set(map(lambda x: x.split('::')[0], full_names_filter)) - + accept_filename_predicate = test_filter.make_py_file_filter(session.config.option.test_filter) + full_test_names_file_path = session.config.option.test_list_path + filenames_filter = None + + if full_test_names_file_path and os.path.exists(full_test_names_file_path): + with open(full_test_names_file_path, 'r') as afile: + # in afile stored 2 dimensional array such that array[modulo_index] contains tests which should be run in this test suite + full_names_filter = set(json.load(afile)[int(session.config.option.modulo_index)]) + filenames_filter = set(map(lambda x: x.split('::')[0], full_names_filter)) + for test_module in self._test_modules: module = LoadedModule.from_parent(name=test_module, parent=session) - if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate): + if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate): yield module - + if os.environ.get('YA_PYTEST_DISABLE_DOCTEST', 'no') == 'no': module = DoctestModule.from_parent(name=test_module, parent=session) if not pytest_ignore_collect(module, session, filenames_filter, accept_filename_predicate): diff --git a/library/python/pytest/plugins/ya.make b/library/python/pytest/plugins/ya.make index c15d6f759d..e1b1e16f02 100644 --- a/library/python/pytest/plugins/ya.make +++ b/library/python/pytest/plugins/ya.make @@ -12,7 +12,7 @@ PY_SRCS( PEERDIR( library/python/filelock library/python/find_root - library/python/testing/filter + library/python/testing/filter ) IF (PYTHON2) diff --git a/library/python/pytest/plugins/ya.py b/library/python/pytest/plugins/ya.py index 1bde03042d..e396516740 100644 --- a/library/python/pytest/plugins/ya.py +++ b/library/python/pytest/plugins/ya.py @@ -29,8 +29,8 @@ import _pytest.skipping from _pytest.warning_types import PytestUnhandledCoroutineWarning -from yatest_lib import test_splitter - +from yatest_lib import test_splitter + try: import resource except ImportError: @@ -54,8 +54,8 @@ import yatest_lib.external as canon import yatest_lib.ya -from library.python.pytest import context - +from library.python.pytest import context + console_logger = logging.getLogger("console") yatest_logger = logging.getLogger("ya.test") @@ -154,7 +154,7 @@ def pytest_addoption(parser): parser.addoption("--test-list-file", action="store", dest="test_list_file") parser.addoption("--modulo", default=1, type=int) parser.addoption("--modulo-index", default=0, type=int) - parser.addoption("--partition-mode", default='SEQUENTIAL', help="Split tests according to partitoin mode") + parser.addoption("--partition-mode", default='SEQUENTIAL', help="Split tests according to partitoin mode") parser.addoption("--split-by-tests", action='store_true', help="Split test execution by tests instead of suites", default=False) parser.addoption("--project-path", action="store", default="", help="path to CMakeList where test is declared") parser.addoption("--build-type", action="store", default="", help="build type") @@ -174,7 +174,7 @@ def pytest_addoption(parser): parser.addoption("--report-deselected", action="store_true", dest="report_deselected", default=False, help="report deselected tests to the trace file") parser.addoption("--pdb-on-sigusr1", action="store_true", default=False, help="setup pdb.set_trace on SIGUSR1") parser.addoption("--test-tool-bin", help="Path to test_tool") - parser.addoption("--test-list-path", dest="test_list_path", action="store", help="path to test list", default="") + parser.addoption("--test-list-path", dest="test_list_path", action="store", help="path to test list", default="") def from_ya_test(): @@ -192,8 +192,8 @@ def pytest_configure(config): config.from_ya_test = from_ya_test() config.test_logs = collections.defaultdict(dict) config.test_metrics = {} - config.suite_metrics = {} - config.configure_timestamp = time.time() + config.suite_metrics = {} + config.configure_timestamp = time.time() context = { "project_path": config.option.project_path, "test_stderr": config.option.test_stderr, @@ -443,53 +443,53 @@ def pytest_collection_modifyitems(items, config): config.hook.pytest_deselected(items=deselected_items) items[:] = filtered_items - def filter_by_full_name(filters): - filter_set = {flt for flt in filters} - filtered_items = [] - deselected_items = [] - for item in items: - if item.nodeid in filter_set: - filtered_items.append(item) - else: - deselected_items.append(item) - - config.hook.pytest_deselected(items=deselected_items) - items[:] = filtered_items - + def filter_by_full_name(filters): + filter_set = {flt for flt in filters} + filtered_items = [] + deselected_items = [] + for item in items: + if item.nodeid in filter_set: + filtered_items.append(item) + else: + deselected_items.append(item) + + config.hook.pytest_deselected(items=deselected_items) + items[:] = filtered_items + # XXX - check to be removed when tests for peerdirs don't run for item in items: if not item.nodeid: item._nodeid = os.path.basename(item.location[0]) - if os.path.exists(config.option.test_list_path): - with open(config.option.test_list_path, 'r') as afile: - chunks = json.load(afile) - filters = chunks[config.option.modulo_index] - filter_by_full_name(filters) - else: - if config.option.test_filter: - filter_items(config.option.test_filter) - partition_mode = config.option.partition_mode - modulo = config.option.modulo - if modulo > 1: - items[:] = sorted(items, key=lambda item: item.nodeid) - modulo_index = config.option.modulo_index - split_by_tests = config.option.split_by_tests - items_by_classes = {} - res = [] - for item in items: - if item.nodeid.count("::") == 2 and not split_by_tests: - class_name = item.nodeid.rsplit("::", 1)[0] - if class_name not in items_by_classes: - items_by_classes[class_name] = [] - res.append(items_by_classes[class_name]) - items_by_classes[class_name].append(item) - else: - res.append([item]) - chunk_items = test_splitter.get_splitted_tests(res, modulo, modulo_index, partition_mode, is_sorted=True) - items[:] = [] - for item in chunk_items: - items.extend(item) - yatest_logger.info("Modulo %s tests are: %s", modulo_index, chunk_items) + if os.path.exists(config.option.test_list_path): + with open(config.option.test_list_path, 'r') as afile: + chunks = json.load(afile) + filters = chunks[config.option.modulo_index] + filter_by_full_name(filters) + else: + if config.option.test_filter: + filter_items(config.option.test_filter) + partition_mode = config.option.partition_mode + modulo = config.option.modulo + if modulo > 1: + items[:] = sorted(items, key=lambda item: item.nodeid) + modulo_index = config.option.modulo_index + split_by_tests = config.option.split_by_tests + items_by_classes = {} + res = [] + for item in items: + if item.nodeid.count("::") == 2 and not split_by_tests: + class_name = item.nodeid.rsplit("::", 1)[0] + if class_name not in items_by_classes: + items_by_classes[class_name] = [] + res.append(items_by_classes[class_name]) + items_by_classes[class_name].append(item) + else: + res.append([item]) + chunk_items = test_splitter.get_splitted_tests(res, modulo, modulo_index, partition_mode, is_sorted=True) + items[:] = [] + for item in chunk_items: + items.extend(item) + yatest_logger.info("Modulo %s tests are: %s", modulo_index, chunk_items) if config.option.mode == yatest_lib.ya.RunMode.Run: for item in items: @@ -543,7 +543,7 @@ def pytest_pyfunc_call(pyfuncitem): @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): - def logreport(report, result, call): + def logreport(report, result, call): test_item = TestItem(report, result, pytest_config.option.test_suffix) if not pytest_config.suite_metrics and context.Ctx.get("YA_PYTEST_START_TIMESTAMP"): pytest_config.suite_metrics["pytest_startup_duration"] = call.start - context.Ctx["YA_PYTEST_START_TIMESTAMP"] @@ -861,10 +861,10 @@ class TraceReportGenerator(object): self.trace('subtest-finished', message) self._test_messages[test_item.nodeid] = message - def dump_suite_metrics(self): + def dump_suite_metrics(self): message = {"metrics": pytest_config.suite_metrics} - self.trace("suite-event", message) - + self.trace("suite-event", message) + def on_error(self, test_item): self.trace('chunk_event', {"errors": [(test_item.status, self._get_comment(test_item))]}) diff --git a/library/python/pytest/ya.make b/library/python/pytest/ya.make index 060c92c313..5666a29077 100644 --- a/library/python/pytest/ya.make +++ b/library/python/pytest/ya.make @@ -10,7 +10,7 @@ PY_SRCS( main.py rewrite.py yatest_tools.py - context.py + context.py ) PEERDIR( diff --git a/library/python/testing/filter/filter.py b/library/python/testing/filter/filter.py index a1642bd052..2a6e19388c 100644 --- a/library/python/testing/filter/filter.py +++ b/library/python/testing/filter/filter.py @@ -1,57 +1,57 @@ -# coding: utf-8 -# TODO move devtools/ya/test/filter.py to library/python/testing/filter/filter.py -import re -import fnmatch -import logging - -logger = logging.getLogger(__name__) -TEST_SUBTEST_SEPARATOR = '::' - -PARSE_TAG_RE = re.compile("([+-]?[\w:]*)") - - -class FilterException(Exception): - mute = True - - -def fix_filter(flt): - if TEST_SUBTEST_SEPARATOR not in flt and "*" not in flt: - # user wants to filter by test module name - flt = flt + TEST_SUBTEST_SEPARATOR + "*" - return flt - - -def escape_for_fnmatch(s): - return s.replace("[", "[").replace("]", "]") - - -def make_py_file_filter(filter_names): - if filter_names is not None: - with_star = [] - wo_star = set() - for flt in filter_names: - flt = flt.split(':')[0] - if '*' in flt: - with_star.append(flt.split('*')[0] + '*') - else: - wo_star.add(flt) - - def predicate(filename): - if filter_names is None: - return True - return filename in wo_star or any([fnmatch.fnmatch(escape_for_fnmatch(filename), escape_for_fnmatch(filter_name)) for filter_name in with_star]) - - return predicate - - -def make_name_filter(filter_names): - filter_names = map(fix_filter, filter_names) - filter_full_names = set() - for name in filter_names: - if '*' not in name: - filter_full_names.add(name) - - def predicate(testname): - return testname in filter_full_names or any([fnmatch.fnmatch(escape_for_fnmatch(testname), escape_for_fnmatch(filter_name)) for filter_name in filter_names]) - - return predicate +# coding: utf-8 +# TODO move devtools/ya/test/filter.py to library/python/testing/filter/filter.py +import re +import fnmatch +import logging + +logger = logging.getLogger(__name__) +TEST_SUBTEST_SEPARATOR = '::' + +PARSE_TAG_RE = re.compile("([+-]?[\w:]*)") + + +class FilterException(Exception): + mute = True + + +def fix_filter(flt): + if TEST_SUBTEST_SEPARATOR not in flt and "*" not in flt: + # user wants to filter by test module name + flt = flt + TEST_SUBTEST_SEPARATOR + "*" + return flt + + +def escape_for_fnmatch(s): + return s.replace("[", "[").replace("]", "]") + + +def make_py_file_filter(filter_names): + if filter_names is not None: + with_star = [] + wo_star = set() + for flt in filter_names: + flt = flt.split(':')[0] + if '*' in flt: + with_star.append(flt.split('*')[0] + '*') + else: + wo_star.add(flt) + + def predicate(filename): + if filter_names is None: + return True + return filename in wo_star or any([fnmatch.fnmatch(escape_for_fnmatch(filename), escape_for_fnmatch(filter_name)) for filter_name in with_star]) + + return predicate + + +def make_name_filter(filter_names): + filter_names = map(fix_filter, filter_names) + filter_full_names = set() + for name in filter_names: + if '*' not in name: + filter_full_names.add(name) + + def predicate(testname): + return testname in filter_full_names or any([fnmatch.fnmatch(escape_for_fnmatch(testname), escape_for_fnmatch(filter_name)) for filter_name in filter_names]) + + return predicate diff --git a/library/python/testing/filter/ya.make b/library/python/testing/filter/ya.make index 22c485d258..a23d145921 100644 --- a/library/python/testing/filter/ya.make +++ b/library/python/testing/filter/ya.make @@ -1,5 +1,5 @@ -PY23_LIBRARY() +PY23_LIBRARY() OWNER(g:yatest) -PY_SRCS(filter.py) - -END() +PY_SRCS(filter.py) + +END() diff --git a/library/python/testing/recipe/ports.py b/library/python/testing/recipe/ports.py index 9f7de1e767..1344a58ac9 100644 --- a/library/python/testing/recipe/ports.py +++ b/library/python/testing/recipe/ports.py @@ -1,33 +1,33 @@ -import os -import sys -import subprocess -import time - -from yatest.common.network import PortManager - - -def __get_port_range(): - port_count = int(sys.argv[1]) - pid_filename = sys.argv[2] - port_manager = PortManager() - start_port = port_manager.get_port_range(None, port_count) - sys.stderr.write(str(start_port) + "\n") - with open(pid_filename, 'w') as afile: - afile.write(str(os.getpid())) - while 1: - time.sleep(1) - - -def get_port_range(port_count=1, pid_filename="recipe_port.pid"): - env = os.environ.copy() - env["Y_PYTHON_ENTRY_POINT"] = "library.python.testing.recipe.ports:__get_port_range" - res = subprocess.Popen([sys.argv[0], str(port_count), pid_filename], env=env, cwd=os.getcwd(), stderr=subprocess.PIPE) - while not os.path.exists(pid_filename): - time.sleep(0.01) - port_start = int(res.stderr.readline()) - return port_start - - -def release_port_range(pid_filename="recipe_port.pid"): - with open(pid_filename, 'r') as afile: - os.kill(int(afile.read()), 9) +import os +import sys +import subprocess +import time + +from yatest.common.network import PortManager + + +def __get_port_range(): + port_count = int(sys.argv[1]) + pid_filename = sys.argv[2] + port_manager = PortManager() + start_port = port_manager.get_port_range(None, port_count) + sys.stderr.write(str(start_port) + "\n") + with open(pid_filename, 'w') as afile: + afile.write(str(os.getpid())) + while 1: + time.sleep(1) + + +def get_port_range(port_count=1, pid_filename="recipe_port.pid"): + env = os.environ.copy() + env["Y_PYTHON_ENTRY_POINT"] = "library.python.testing.recipe.ports:__get_port_range" + res = subprocess.Popen([sys.argv[0], str(port_count), pid_filename], env=env, cwd=os.getcwd(), stderr=subprocess.PIPE) + while not os.path.exists(pid_filename): + time.sleep(0.01) + port_start = int(res.stderr.readline()) + return port_start + + +def release_port_range(pid_filename="recipe_port.pid"): + with open(pid_filename, 'r') as afile: + os.kill(int(afile.read()), 9) diff --git a/library/python/testing/recipe/ya.make b/library/python/testing/recipe/ya.make index dd323aa245..4698546e42 100644 --- a/library/python/testing/recipe/ya.make +++ b/library/python/testing/recipe/ya.make @@ -5,14 +5,14 @@ OWNER( PY23_LIBRARY() -PY_SRCS( - __init__.py - ports.py -) +PY_SRCS( + __init__.py + ports.py +) PEERDIR( contrib/python/ipdb - library/python/testing/yatest_common + library/python/testing/yatest_common library/python/testing/yatest_lib ) diff --git a/library/python/testing/ya.make b/library/python/testing/ya.make index 883bc8d7ab..002b406a61 100644 --- a/library/python/testing/ya.make +++ b/library/python/testing/ya.make @@ -4,7 +4,7 @@ RECURSE( behave deprecated fake_ya_package - filter + filter gtest gtest/test gtest/test/gtest diff --git a/library/python/testing/yatest_common/yatest/common/path.py b/library/python/testing/yatest_common/yatest/common/path.py index 6fed7dda8a..313c67ef39 100644 --- a/library/python/testing/yatest_common/yatest/common/path.py +++ b/library/python/testing/yatest_common/yatest/common/path.py @@ -59,7 +59,7 @@ def copytree(src, dst, symlinks=False, ignore=None, postprocessing=None): def get_unique_file_path(dir_path, file_pattern, create_file=True, max_suffix=10000): def atomic_file_create(path): try: - fd = os.open(path, os.O_CREAT | os.O_EXCL, 0o644) + fd = os.open(path, os.O_CREAT | os.O_EXCL, 0o644) os.close(fd) return True except OSError as e: diff --git a/library/python/testing/yatest_common/yatest/common/process.py b/library/python/testing/yatest_common/yatest/common/process.py index a8bcc21f51..58e12d8f69 100644 --- a/library/python/testing/yatest_common/yatest/common/process.py +++ b/library/python/testing/yatest_common/yatest/common/process.py @@ -79,12 +79,12 @@ class InvalidExecutionStateError(Exception): pass -class SignalInterruptionError(Exception): - def __init__(self, message=None): - super(SignalInterruptionError, self).__init__(message) - self.res = None - - +class SignalInterruptionError(Exception): + def __init__(self, message=None): + super(SignalInterruptionError, self).__init__(message) + self.res = None + + class InvalidCommandError(Exception): pass @@ -284,7 +284,7 @@ class _Execution(object): def wait(self, check_exit_code=True, timeout=None, on_timeout=None): def _wait(): finished = None - interrupted = False + interrupted = False try: if hasattr(os, "wait4"): try: @@ -321,12 +321,12 @@ class _Execution(object): yatest_logger.debug("Process resource usage is not available as process finished before wait4 was called") else: raise - except SignalInterruptionError: - interrupted = True - raise + except SignalInterruptionError: + interrupted = True + raise finally: - if not interrupted: - self._process.wait() # this has to be here unconditionally, so that all process properties are set + if not interrupted: + self._process.wait() # this has to be here unconditionally, so that all process properties are set if not finished: finished = time.time() diff --git a/library/python/testing/yatest_common/yatest/common/runtime.py b/library/python/testing/yatest_common/yatest/common/runtime.py index e55e193446..80f8f70db4 100644 --- a/library/python/testing/yatest_common/yatest/common/runtime.py +++ b/library/python/testing/yatest_common/yatest/common/runtime.py @@ -242,11 +242,11 @@ def c_compiler_path(): return os.environ.get("YA_CC") -def get_yt_hdd_path(path=None): - if 'HDD_PATH' in os.environ: - return _join_path(os.environ['HDD_PATH'], path) - - +def get_yt_hdd_path(path=None): + if 'HDD_PATH' in os.environ: + return _join_path(os.environ['HDD_PATH'], path) + + def cxx_compiler_path(): """ Get path to the gdb diff --git a/library/python/testing/yatest_lib/test_splitter.py b/library/python/testing/yatest_lib/test_splitter.py index acbcd4300e..2f2ec39a6e 100644 --- a/library/python/testing/yatest_lib/test_splitter.py +++ b/library/python/testing/yatest_lib/test_splitter.py @@ -18,66 +18,66 @@ def flatten_tests(test_classes): return tests -def get_sequential_chunk(tests, modulo, modulo_index, is_sorted=False): +def get_sequential_chunk(tests, modulo, modulo_index, is_sorted=False): """ - >>> get_sequential_chunk(range(10), 4, 0) + >>> get_sequential_chunk(range(10), 4, 0) [0, 1, 2] - >>> get_sequential_chunk(range(10), 4, 1) + >>> get_sequential_chunk(range(10), 4, 1) [3, 4, 5] - >>> get_sequential_chunk(range(10), 4, 2) - [6, 7] - >>> get_sequential_chunk(range(10), 4, 3) - [8, 9] - >>> get_sequential_chunk(range(10), 4, 4) + >>> get_sequential_chunk(range(10), 4, 2) + [6, 7] + >>> get_sequential_chunk(range(10), 4, 3) + [8, 9] + >>> get_sequential_chunk(range(10), 4, 4) [] - >>> get_sequential_chunk(range(10), 4, 5) + >>> get_sequential_chunk(range(10), 4, 5) [] """ - if not is_sorted: - tests = sorted(tests) - chunk_size = len(tests) // modulo - not_used = len(tests) % modulo - shift = chunk_size + (modulo_index < not_used) - start = chunk_size * modulo_index + min(modulo_index, not_used) + if not is_sorted: + tests = sorted(tests) + chunk_size = len(tests) // modulo + not_used = len(tests) % modulo + shift = chunk_size + (modulo_index < not_used) + start = chunk_size * modulo_index + min(modulo_index, not_used) end = start + shift - return [] if end > len(tests) else tests[start:end] + return [] if end > len(tests) else tests[start:end] -def get_shuffled_chunk(tests, modulo, modulo_index, is_sorted=False): - """ - >>> get_shuffled_chunk(range(10), 4, 0) - [0, 4, 8] - >>> get_shuffled_chunk(range(10), 4, 1) - [1, 5, 9] - >>> get_shuffled_chunk(range(10), 4, 2) - [2, 6] - >>> get_shuffled_chunk(range(10), 4, 3) - [3, 7] - >>> get_shuffled_chunk(range(10), 4, 4) - [] - >>> get_shuffled_chunk(range(10), 4, 5) - [] - """ - if not is_sorted: - tests = sorted(tests) - result_tests = [] - for i, test in enumerate(tests): - if i % modulo == modulo_index: - result_tests.append(test) - return result_tests - - -def get_splitted_tests(test_entities, modulo, modulo_index, partition_mode, is_sorted=False): - if partition_mode == 'SEQUENTIAL': - return get_sequential_chunk(test_entities, modulo, modulo_index, is_sorted) - elif partition_mode == 'MODULO': - return get_shuffled_chunk(test_entities, modulo, modulo_index, is_sorted) - else: - raise ValueError("detected unknown partition mode: {}".format(partition_mode)) - - -def filter_tests_by_modulo(test_classes, modulo, modulo_index, split_by_tests, partition_mode="SEQUENTIAL"): +def get_shuffled_chunk(tests, modulo, modulo_index, is_sorted=False): """ + >>> get_shuffled_chunk(range(10), 4, 0) + [0, 4, 8] + >>> get_shuffled_chunk(range(10), 4, 1) + [1, 5, 9] + >>> get_shuffled_chunk(range(10), 4, 2) + [2, 6] + >>> get_shuffled_chunk(range(10), 4, 3) + [3, 7] + >>> get_shuffled_chunk(range(10), 4, 4) + [] + >>> get_shuffled_chunk(range(10), 4, 5) + [] + """ + if not is_sorted: + tests = sorted(tests) + result_tests = [] + for i, test in enumerate(tests): + if i % modulo == modulo_index: + result_tests.append(test) + return result_tests + + +def get_splitted_tests(test_entities, modulo, modulo_index, partition_mode, is_sorted=False): + if partition_mode == 'SEQUENTIAL': + return get_sequential_chunk(test_entities, modulo, modulo_index, is_sorted) + elif partition_mode == 'MODULO': + return get_shuffled_chunk(test_entities, modulo, modulo_index, is_sorted) + else: + raise ValueError("detected unknown partition mode: {}".format(partition_mode)) + + +def filter_tests_by_modulo(test_classes, modulo, modulo_index, split_by_tests, partition_mode="SEQUENTIAL"): + """ >>> test_classes = {x: [x] for x in range(20)} >>> filter_tests_by_modulo(test_classes, 4, 0, False) {0: [0], 1: [1], 2: [2], 3: [3], 4: [4]} @@ -92,11 +92,11 @@ def filter_tests_by_modulo(test_classes, modulo, modulo_index, split_by_tests, p {8: [8], 9: [9], 5: [5], 6: [6], 7: [7]} """ if split_by_tests: - tests = get_splitted_tests(flatten_tests(test_classes), modulo, modulo_index, partition_mode) + tests = get_splitted_tests(flatten_tests(test_classes), modulo, modulo_index, partition_mode) test_classes = collections.defaultdict(list) for class_name, test_name in tests: test_classes[class_name].append(test_name) return test_classes else: - target_classes = get_splitted_tests(test_classes, modulo, modulo_index, partition_mode) + target_classes = get_splitted_tests(test_classes, modulo, modulo_index, partition_mode) return {class_name: test_classes[class_name] for class_name in target_classes} diff --git a/library/python/testing/yatest_lib/tests/test_testsplitter.py b/library/python/testing/yatest_lib/tests/test_testsplitter.py index 394bfe5a74..87842e0ceb 100644 --- a/library/python/testing/yatest_lib/tests/test_testsplitter.py +++ b/library/python/testing/yatest_lib/tests/test_testsplitter.py @@ -1,103 +1,103 @@ -# coding: utf-8 -from yatest_lib import test_splitter - - -def get_chunks(tests, modulo, mode): - chunks = [] - if mode == "MODULO": - for modulo_index in range(modulo): - chunks.append(test_splitter.get_shuffled_chunk(tests, modulo, modulo_index)) - elif mode == "SEQUENTIAL": - for modulo_index in range(modulo): - chunks.append(test_splitter.get_sequential_chunk(tests, modulo, modulo_index)) - else: - raise ValueError("no such mode") - return chunks - - -def check_not_intersect(chunk_list): - test_set = set() - total_size = 0 - for tests in chunk_list: - total_size += len(tests) - test_set.update(tests) - return total_size == len(test_set) - - -def check_max_diff(chunk_list): - return max(map(len, chunk_list)) - min(map(len, chunk_list)) - - -def test_lot_of_chunks(): - for chunk_count in range(10, 20): - for tests_count in range(chunk_count): - chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) <= 1 - assert chunks.count([]) == chunk_count - tests_count - assert len(chunks) == chunk_count - chunks = get_chunks(range(tests_count), chunk_count, "MODULO") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) <= 1 - assert chunks.count([]) == chunk_count - tests_count - assert len(chunks) == chunk_count - - -def test_lot_of_tests(): - for tests_count in range(10, 20): - for chunk_count in range(2, tests_count): - chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) <= 1 - assert len(chunks) == chunk_count - chunks = get_chunks(range(tests_count), chunk_count, "MODULO") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) <= 1 - assert len(chunks) == chunk_count - - -def prime_chunk_count(): - for chunk_count in [7, 11, 13, 17, 23, 29]: - for tests_count in range(chunk_count): - chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) <= 1 - assert len(chunks) == chunk_count - chunks = get_chunks(range(tests_count), chunk_count, "MODULO") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) <= 1 - assert len(chunks) == chunk_count - - -def get_divisors(number): - divisors = [] - for d in range(1, number + 1): - if number % d == 0: - divisors.append(d) - return divisors - - -def equal_chunks(): - for chunk_count in range(12, 31): - for tests_count in get_divisors(chunk_count): - chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) == 0 - assert len(chunks) == chunk_count - chunks = get_chunks(range(tests_count), chunk_count, "MODULO") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) == 0 - assert len(chunks) == chunk_count - - -def chunk_count_equal_tests_count(): - for chunk_count in range(10, 20): - tests_count = chunk_count - chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) <= 1 - assert len(chunks) == chunk_count - chunks = get_chunks(range(tests_count), chunk_count, "MODULO") - assert check_not_intersect(chunks) - assert check_max_diff(chunks) <= 1 - assert len(chunks) == chunk_count +# coding: utf-8 +from yatest_lib import test_splitter + + +def get_chunks(tests, modulo, mode): + chunks = [] + if mode == "MODULO": + for modulo_index in range(modulo): + chunks.append(test_splitter.get_shuffled_chunk(tests, modulo, modulo_index)) + elif mode == "SEQUENTIAL": + for modulo_index in range(modulo): + chunks.append(test_splitter.get_sequential_chunk(tests, modulo, modulo_index)) + else: + raise ValueError("no such mode") + return chunks + + +def check_not_intersect(chunk_list): + test_set = set() + total_size = 0 + for tests in chunk_list: + total_size += len(tests) + test_set.update(tests) + return total_size == len(test_set) + + +def check_max_diff(chunk_list): + return max(map(len, chunk_list)) - min(map(len, chunk_list)) + + +def test_lot_of_chunks(): + for chunk_count in range(10, 20): + for tests_count in range(chunk_count): + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert chunks.count([]) == chunk_count - tests_count + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert chunks.count([]) == chunk_count - tests_count + assert len(chunks) == chunk_count + + +def test_lot_of_tests(): + for tests_count in range(10, 20): + for chunk_count in range(2, tests_count): + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + + +def prime_chunk_count(): + for chunk_count in [7, 11, 13, 17, 23, 29]: + for tests_count in range(chunk_count): + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + + +def get_divisors(number): + divisors = [] + for d in range(1, number + 1): + if number % d == 0: + divisors.append(d) + return divisors + + +def equal_chunks(): + for chunk_count in range(12, 31): + for tests_count in get_divisors(chunk_count): + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) == 0 + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) == 0 + assert len(chunks) == chunk_count + + +def chunk_count_equal_tests_count(): + for chunk_count in range(10, 20): + tests_count = chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "SEQUENTIAL") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count + chunks = get_chunks(range(tests_count), chunk_count, "MODULO") + assert check_not_intersect(chunks) + assert check_max_diff(chunks) <= 1 + assert len(chunks) == chunk_count diff --git a/library/python/testing/yatest_lib/tests/ya.make b/library/python/testing/yatest_lib/tests/ya.make index 8586c6ef7d..6670106533 100644 --- a/library/python/testing/yatest_lib/tests/ya.make +++ b/library/python/testing/yatest_lib/tests/ya.make @@ -1,14 +1,14 @@ OWNER(g:yatest) - + PY23_TEST() - -PEERDIR( - library/python/testing/yatest_lib -) - -TEST_SRCS( + +PEERDIR( + library/python/testing/yatest_lib +) + +TEST_SRCS( test_external.py - test_testsplitter.py -) - -END() + test_testsplitter.py +) + +END() diff --git a/library/python/testing/yatest_lib/ya.make b/library/python/testing/yatest_lib/ya.make index 342bae82ba..9b1d5224ea 100644 --- a/library/python/testing/yatest_lib/ya.make +++ b/library/python/testing/yatest_lib/ya.make @@ -6,7 +6,7 @@ PY_SRCS( NAMESPACE yatest_lib external.py - test_splitter.py + test_splitter.py tools.py ya.py ) @@ -22,5 +22,5 @@ IF(PYTHON2) ENDIF() END() - -RECURSE_FOR_TESTS(tests) + +RECURSE_FOR_TESTS(tests) |