diff options
author | prettyboy <prettyboy@yandex-team.com> | 2023-09-08 10:08:57 +0300 |
---|---|---|
committer | prettyboy <prettyboy@yandex-team.com> | 2023-09-08 10:39:21 +0300 |
commit | 329f805999a3b41e406959a17cf35ab193ef05a5 (patch) | |
tree | 9a3059df4d544b6c9d6f474344e52f65bd13b4c1 /tools | |
parent | 02ea6261088be81bbc455933cecf8b41726946c1 (diff) | |
download | ydb-329f805999a3b41e406959a17cf35ab193ef05a5.tar.gz |
Revert commit rXXXXXX,[build/plugins/ytest] Allow prebuilt linters for opensource
Diffstat (limited to 'tools')
-rw-r--r-- | tools/black_linter/ya.make | 11 | ||||
-rw-r--r-- | tools/flake8_linter/bin/__main__.py | 190 | ||||
-rw-r--r-- | tools/flake8_linter/bin/tests/stub/__main__.py | 51 | ||||
-rw-r--r-- | tools/flake8_linter/bin/tests/stub/ya.make | 9 | ||||
-rw-r--r-- | tools/flake8_linter/bin/tests/test_flake8_ver.py | 30 | ||||
-rw-r--r-- | tools/flake8_linter/bin/tests/test_migrations.py | 164 | ||||
-rw-r--r-- | tools/flake8_linter/bin/tests/test_noqa.py | 73 | ||||
-rw-r--r-- | tools/flake8_linter/bin/tests/test_report.py | 101 | ||||
-rw-r--r-- | tools/flake8_linter/bin/tests/util.py | 151 | ||||
-rw-r--r-- | tools/flake8_linter/bin/tests/ya.make | 29 | ||||
-rw-r--r-- | tools/flake8_linter/bin/ya.make | 20 | ||||
-rw-r--r-- | tools/flake8_linter/ya.make | 15 |
12 files changed, 0 insertions, 844 deletions
diff --git a/tools/black_linter/ya.make b/tools/black_linter/ya.make deleted file mode 100644 index b5607dc159..0000000000 --- a/tools/black_linter/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -IF (USE_PREBUILT_TOOLS OR OPENSOURCE) - INCLUDE(${ARCADIA_ROOT}/build/prebuilt/tools/black_linter/ya.make.prebuilt) -ENDIF() - -IF (NOT PREBUILT) - INCLUDE(${ARCADIA_ROOT}/tools/black_linter/bin/ya.make) -ENDIF() - -RECURSE( - bin -) diff --git a/tools/flake8_linter/bin/__main__.py b/tools/flake8_linter/bin/__main__.py deleted file mode 100644 index 77a8edc881..0000000000 --- a/tools/flake8_linter/bin/__main__.py +++ /dev/null @@ -1,190 +0,0 @@ -import configparser -import hashlib -import itertools -import io -import logging -import os -import re -import shutil -import subprocess -import sys -import tempfile -from collections import defaultdict -from typing import Generator - -from devtools.ya.test.programs.test_tool.lib.migrations_config import load_yaml_config, MigrationsConfig -from library.python.testing.custom_linter_util import linter_params, reporter -from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE - -logger = logging.getLogger(__name__) - -ALLOWED_IGNORES = {"F401"} -# Supports both default and pylint formats -FLAKE_LINE_RE = r"^(.*?):(\d+):(\d+:)? \[(\w+\d+)\] (.*)" -FLAKE8_CONFIG_INDEX = 0 -MIGRATIONS_CONFIG_INDEX = 1 - - -def get_flake8_bin(params) -> str: - if params.lint_name == "py2_flake8": - flake8_root = params.global_resources[FLAKE8_PY2_RESOURCE] - elif params.lint_name == "py3_flake8": - flake8_root = params.global_resources[FLAKE8_PY3_RESOURCE] - else: - raise RuntimeError("Unexpected lint name: {}".format(params.lint_name)) - return os.path.join(flake8_root, "flake8") - - -def get_migrations_config(params) -> MigrationsConfig: - if params.extra_params.get("DISABLE_FLAKE8_MIGRATIONS", "no") == "yes": - return MigrationsConfig() - config_path = os.getenv("_YA_TEST_FLAKE8_CONFIG", params.configs[MIGRATIONS_CONFIG_INDEX]) - if not config_path: - return MigrationsConfig() - else: - logger.debug("Loading flake8 migrations: %s", config_path) - migrations = load_yaml_config(config_path) - logger.debug("Building migration config") - return MigrationsConfig(migrations) - - -def get_flake8_config( - flake8_config: str, migrations_config: MigrationsConfig, source_root: str, file_path: str -) -> str | None: - arc_rel_file_path = os.path.relpath(file_path, source_root) - if migrations_config.is_skipped(arc_rel_file_path): - return None - exceptions = migrations_config.get_exceptions(arc_rel_file_path) - if exceptions: - logger.info("Ignore flake8 exceptions %s for file %s", str(list(exceptions)), arc_rel_file_path) - - if os.path.basename(file_path) == "__init__.py": - exceptions |= get_noqa_exceptions(file_path) - - if exceptions: - new_config = configparser.ConfigParser() - new_config.read(flake8_config) # https://bugs.python.org/issue16058 Why don't use deepcopy - new_config["flake8"]["ignore"] += "\n" + "\n".join(x + "," for x in sorted(exceptions)) - - config_stream = io.StringIO() - new_config.write(config_stream) - config_hash = hashlib.md5(config_stream.getvalue().encode()).hexdigest() - config_path = config_hash + ".config" - if not os.path.exists(config_path): - with open(config_path, "w") as f: - f.write(config_stream.getvalue()) - return config_path - else: - return flake8_config - - -def get_noqa_exceptions(file_path: str) -> set: - additional_exceptions = get_file_ignores(file_path) - validate_exceptions(additional_exceptions) - return additional_exceptions & ALLOWED_IGNORES - - -def get_file_ignores(file_path: str) -> set: - file_ignore_regex = re.compile(r"#\s*flake8\s+noqa:\s*(.*)") - with open(file_path) as afile: - # looking for ignores only in the first 3 lines - for line in itertools.islice(afile, 3): - if match := file_ignore_regex.search(line): - ignores = match.group(1).strip() - if ignores: - ignores = re.split(r"\s*,\s*", ignores) - return set(ignores) - return set() - - -def validate_exceptions(exceptions: set) -> None: - if exceptions - ALLOWED_IGNORES: - logger.error( - "Disabling %s checks. Only %s can be suppressed in the __init__.py files using # flake8 noqa", - str(list(exceptions - ALLOWED_IGNORES)), - str(list(ALLOWED_IGNORES)), - ) - - -def run_flake8_for_dir(flake8_bin: str, source_root: str, config: str, check_files: list[str]) -> dict[str, list[str]]: - with tempfile.TemporaryDirectory() as temp_dir: - logger.debug("flake8 temp dir: %s", temp_dir) - for f in check_files: - copy_file_path = os.path.join(temp_dir, os.path.relpath(f, source_root)) - os.makedirs(os.path.dirname(copy_file_path), exist_ok=True) - shutil.copyfile(f, copy_file_path) - flake8_res = run_flake8(flake8_bin, temp_dir, config) - return get_flake8_results(flake8_res, source_root, temp_dir) - - -def run_flake8(flake8_bin: str, dir_path: str, config: str) -> list[str]: - cmd = [flake8_bin, dir_path, config] - res = subprocess.run(cmd, capture_output=True, encoding="utf8", errors="replace") - if res.stderr: - logger.debug("flake8 stderr: %s", res.stderr) - return res.stdout.split("\n") if res.returncode else [] - - -def get_flake8_results(flake8_res: list[str], source_root: str, temp_dir: str) -> dict[str, list[str]]: - flake8_errors_map = defaultdict(list) - for line in iterate_over_results(flake8_res): - match = re.match(FLAKE_LINE_RE, line) - if not match: - raise RuntimeError("Cannot parse flake8 output line: '{}'".format(line)) - file_path, row, col_with_sep, code, text = match.groups() - file_path = file_path.replace(temp_dir, source_root) - if col_with_sep is None: - col_with_sep = "" - colorized_line = f"[[unimp]]{file_path}[[rst]]:[[alt2]]{row}[[rst]]:[[alt2]]{col_with_sep}[[rst]] [[[alt1]]{code}[[rst]]] [[bad]]{text}[[rst]]" - flake8_errors_map[file_path].append(colorized_line) - return flake8_errors_map - - -def iterate_over_results(flake8_res: list[str]) -> Generator[str, None, None]: - to_skip = {"[[bad]]", "[[rst]]"} - for line in flake8_res: - if line and line not in to_skip: - yield line - - -def main(): - params = linter_params.get_params() - logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format="%(asctime)s: %(levelname)s: %(message)s") - - flake8_bin = get_flake8_bin(params) - flake8_config = params.configs[FLAKE8_CONFIG_INDEX] - migrations_config = get_migrations_config(params) - source_root = params.source_root - - logger.debug("Constructing flake8 config") - config_map = defaultdict(list) - report = reporter.LintReport() - - skipped_files = set() - for file_path in params.files: - config_path = get_flake8_config(flake8_config, migrations_config, source_root, file_path) - if config_path: - config_map[config_path].append(file_path) - else: - skipped_files.add(file_path) - - logger.debug("Configuration:\n%s", str(config_map)) - - flake8_errors_map = {} - for config_path, check_files in config_map.items(): - flake8_errors_map.update(run_flake8_for_dir(flake8_bin, source_root, config_path, check_files)) - - report = reporter.LintReport() - for file_path in params.files: - if file_path in skipped_files: - report.add(file_path, reporter.LintStatus.SKIPPED, "Skipped by config") - elif file_path in flake8_errors_map: - message = "\n".join(flake8_errors_map[file_path]) - report.add(file_path, reporter.LintStatus.FAIL, message) - else: - report.add(file_path, reporter.LintStatus.GOOD) - report.dump(params.report_file) - - -if __name__ == "__main__": - main() diff --git a/tools/flake8_linter/bin/tests/stub/__main__.py b/tools/flake8_linter/bin/tests/stub/__main__.py deleted file mode 100644 index 3bbfcf424b..0000000000 --- a/tools/flake8_linter/bin/tests/stub/__main__.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -flake8 emulator. Does the following: -- read config file (the name is specified in _FLAKE8_STUB_CONFIG env variable) -- gather launch info and put it into the report file -- print test data to stdout -""" - -import json -import os -import sys - - -STUB_CONFIG_ENV_VAR_NAME = "_FLAKE8_STUB_CONFIG" - - -def main(): - flake8_bin, test_dir, flake8_config = sys.argv - stub_config_file = os.getenv(STUB_CONFIG_ENV_VAR_NAME) - with open(stub_config_file) as f: - stub_config = json.load(f) - - stub_output = stub_config["output"] - launch_report_file = stub_config["report_file"] - - launch_report = get_launch_report(flake8_bin, test_dir, flake8_config) - with open(launch_report_file, "a") as f: - json.dump(launch_report, f) - f.write("\n") - - if stub_output: - sys.stdout.write(stub_output.format(test_dir=test_dir)) - return 1 - else: - return 0 - - -def get_launch_report(flake8_bin, test_dir, flake8_config): - rel_file_paths = [] - for root, _, files in os.walk(test_dir): - rel_file_paths += [os.path.relpath(os.path.join(root, f), test_dir) for f in files] - with open(flake8_config) as f: - config_data = f.read() - return { - "flake8_bin": flake8_bin, - "rel_file_paths": rel_file_paths, - "config_data": config_data, - } - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/flake8_linter/bin/tests/stub/ya.make b/tools/flake8_linter/bin/tests/stub/ya.make deleted file mode 100644 index 8c24917942..0000000000 --- a/tools/flake8_linter/bin/tests/stub/ya.make +++ /dev/null @@ -1,9 +0,0 @@ -PY3_PROGRAM(flake8) - -STYLE_PYTHON() - -PY_SRCS( - __main__.py -) - -END() diff --git a/tools/flake8_linter/bin/tests/test_flake8_ver.py b/tools/flake8_linter/bin/tests/test_flake8_ver.py deleted file mode 100644 index d2760a8c7d..0000000000 --- a/tools/flake8_linter/bin/tests/test_flake8_ver.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import pytest - -from . import util -from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE - - -@pytest.mark.parametrize( - "lint_name, global_resource_var_name", - [ - ("py2_flake8", FLAKE8_PY2_RESOURCE), - ("py3_flake8", FLAKE8_PY3_RESOURCE), - ], -) -def test_flake8_version(lint_name, global_resource_var_name): - test_file = "project/test.py" - runner = util.LinterRunner(lint_name) - runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file]) - run_result = runner.run_test([test_file]) - expected_flake8_bin = os.path.join(runner.flake8_path(global_resource_var_name), "flake8") - assert run_result.flake8_launches[0].flake8_bin == expected_flake8_bin - - -def test_raise_on_incorrect_lint_name(): - test_file = "project/test.py" - runner = util.LinterRunner("strange_lint_name") - runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file]) - run_result = runner.run_test([test_file]) - assert run_result.linter_run_result.returncode != 0 - assert "Unexpected lint name" in run_result.linter_run_result.stderr diff --git a/tools/flake8_linter/bin/tests/test_migrations.py b/tools/flake8_linter/bin/tests/test_migrations.py deleted file mode 100644 index 1e77d265be..0000000000 --- a/tools/flake8_linter/bin/tests/test_migrations.py +++ /dev/null @@ -1,164 +0,0 @@ -import pytest -from configparser import ConfigParser - -from . import util - -FLAKE8_CONFIG_DATA = """ - [flake8] - select = E, W, F - ignore = - E122, - E743, - F403, - W605, -""" -MIGRATION_CONFIG_DATA = """ - flake8: - F401: - ignore: - - F401 - prefixes: - - project401 - SKIP: - ignore: - - "*" - prefixes: - - project_skip -""" -FILE_WITHOUT_EXCEPTIONS = "project/file1.py" -FILE_IGNORE_F401 = "project401/file2.py" -FILE_SKIPPED = "project_skip/file3.py" - - -@pytest.mark.parametrize( - "test_file, added_ignore, disable_migrations", - [ - (FILE_WITHOUT_EXCEPTIONS, "", False), - (FILE_IGNORE_F401, "\nF401,", False), - (FILE_IGNORE_F401, "", True), - ], -) -def test_ignore(test_file, added_ignore, disable_migrations): - test_files = [test_file] - - runner = util.LinterRunner() - runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA) - runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA) - runner.create_source_tree(test_files) - - disable_migrations_params = {"extra_params": {"DISABLE_FLAKE8_MIGRATIONS": "yes"}} if disable_migrations else {} - run_result = runner.run_test(test_files, custom_params=disable_migrations_params) - - assert len(run_result.flake8_launches) == 1 - - launch = run_result.flake8_launches[0] - got_config = ConfigParser() - got_config.read_string(launch.config_data) - expected_config = ConfigParser() - expected_config.read_string(FLAKE8_CONFIG_DATA) - expected_config["flake8"]["ignore"] += added_ignore - - util.assert_configs(got_config, expected_config) - - assert launch.rel_file_paths == test_files - - -def test_skipped(): - test_files = [FILE_SKIPPED] - - runner = util.LinterRunner() - runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA) - runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA) - runner.create_source_tree(test_files) - - run_result = runner.run_test(test_files) - - assert len(run_result.flake8_launches) == 0 - - abs_test_file_path = runner.abs_source_file_path(FILE_SKIPPED) - assert run_result.report_data["report"][abs_test_file_path]["status"] == "SKIPPED" - - -def test_group_files_by_config(): - test_files = [FILE_WITHOUT_EXCEPTIONS, FILE_IGNORE_F401] - - runner = util.LinterRunner() - runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA) - runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA) - runner.create_source_tree(test_files) - - run_result = runner.run_test(test_files) - - assert len(run_result.flake8_launches) == 2 - - for launch in run_result.flake8_launches: - rel_file_paths = launch.rel_file_paths - got_config = ConfigParser() - got_config.read_string(launch.config_data) - # Relaxed check if config is matched with a checked file - # Thorough config check is done in test_ignore() - if rel_file_paths == [FILE_WITHOUT_EXCEPTIONS]: - assert "F401" not in got_config["flake8"]["ignore"] - elif rel_file_paths == [FILE_IGNORE_F401]: - assert "F401" in got_config["flake8"]["ignore"] - else: - pytest.fail("Unexpected file paths passed to flake8 binary: {}".format(rel_file_paths)) - - -@pytest.mark.parametrize( - "migrations_file, expected_ignore", - [ - (None, "F777"), - ("", None), - ("build/config/other_migration.yaml", "F888"), - ], -) -def test_migration_file_from_env(migrations_file, expected_ignore): - # Env var _YA_TEST_FLAKE8_CONFIG overrides file name from configs parameter. _YA_TEST_FLAKE8_CONFIG: - # - is not defined - use migrations file from configs parameter - # - is empty - don't use migrations file at all - # - not empty - use variable value as migrations file name - config_migrations = """ - flake8: - ignore: - ignore: - - F777 - prefixes: - - project - """ - test_files = ["project/test.py"] - - runner = util.LinterRunner() - runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA) - runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, config_migrations) - runner.create_source_tree(test_files) - env = {} - if migrations_file is not None: - if migrations_file: - env_var_migrations = """ - flake8: - ignore: - ignore: - - F888 - prefixes: - - project - """ - runner.create_source_file(migrations_file, env_var_migrations) - env_var_value = runner.abs_source_file_path(migrations_file) - else: - env_var_value = "" - env["_YA_TEST_FLAKE8_CONFIG"] = env_var_value - - run_result = runner.run_test(test_files, env=env) - - assert len(run_result.flake8_launches) == 1 - - launch = run_result.flake8_launches[0] - got_config = ConfigParser() - got_config.read_string(launch.config_data) - ignores = got_config["flake8"]["ignore"] - if expected_ignore: - assert expected_ignore in ignores - else: - assert "F777" not in ignores - assert "F888" not in ignores diff --git a/tools/flake8_linter/bin/tests/test_noqa.py b/tools/flake8_linter/bin/tests/test_noqa.py deleted file mode 100644 index 1f7d536fe7..0000000000 --- a/tools/flake8_linter/bin/tests/test_noqa.py +++ /dev/null @@ -1,73 +0,0 @@ -import io -import pytest -from configparser import ConfigParser - -from . import util - -FLAKE8_CONFIG_DATA = """ - [flake8] - select = E, W, F - ignore = - E122, -""" - - -@pytest.mark.parametrize( - "file_name, noqa_line_no, is_scanned", - [ - ("__init__.py", 1, True), - ("__init__.py", 3, True), - ("__init__.py", 4, False), - ("not_init.py", 1, False), - ], -) -def test_scanned_line_count(file_name, noqa_line_no, is_scanned): - test_file = "project/" + file_name - - test_file_data = io.StringIO() - for lno in range(1, 10): - test_file_data.write("pass") - if lno == noqa_line_no: - test_file_data.write(" # flake8 noqa: F401") - test_file_data.write("\n") - - runner = util.LinterRunner() - runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA) - runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, "") - runner.create_source_file(test_file, test_file_data.getvalue()) - - run_result = runner.run_test([test_file]) - - assert len(run_result.flake8_launches) == 1 - - launch = run_result.flake8_launches[0] - got_config = ConfigParser() - got_config.read_string(launch.config_data) - if is_scanned: - assert "F401" in got_config["flake8"]["ignore"] - else: - assert "F401" not in got_config["flake8"]["ignore"] - - -def test_not_F401(): - test_file = "project/__init__.py" - - test_file_data = """ - pass # flake8 noqa: F777, F401 - """ - - runner = util.LinterRunner() - runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA) - runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, "") - runner.create_source_file(test_file, test_file_data) - - run_result = runner.run_test([test_file]) - - assert len(run_result.flake8_launches) == 1 - - launch = run_result.flake8_launches[0] - got_config = ConfigParser() - got_config.read_string(launch.config_data) - assert "F401" in got_config["flake8"]["ignore"] - assert "F777" not in got_config["flake8"]["ignore"] - assert "Disabling ['F777'] checks" in run_result.linter_run_result.stdout diff --git a/tools/flake8_linter/bin/tests/test_report.py b/tools/flake8_linter/bin/tests/test_report.py deleted file mode 100644 index 9304fe86b3..0000000000 --- a/tools/flake8_linter/bin/tests/test_report.py +++ /dev/null @@ -1,101 +0,0 @@ -import pytest - -from . import util - -FLAKE8_CONFIG_DATA = """ - [flake8] - select = E, W, F - ignore = - E122, -""" - - -def test_no_errors(): - test_file = "project/test.py" - runner = util.LinterRunner() - runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file]) - - run_result = runner.run_test([test_file]) - - abs_test_file = runner.abs_source_file_path(test_file) - file_report = run_result.report_data["report"][abs_test_file] - assert file_report["status"] == "GOOD" - assert file_report["message"] == "" - assert file_report["elapsed"] >= 0.0 - - -def test_skip_markup(): - test_file = "project/test.py" - flake8_result = """ - [[bad]] - [[rst]] - """ - - runner = util.LinterRunner() - runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file]) - - run_result = runner.run_test([test_file], flake8_result=flake8_result) - - abs_test_file = runner.abs_source_file_path(test_file) - file_report = run_result.report_data["report"][abs_test_file] - assert file_report["status"] == "GOOD" - assert file_report["message"] == "" - assert file_report["elapsed"] >= 0.0 - - -@pytest.mark.parametrize( - "errors", - [ - [("10", "F401", "Error with row number only")], - [("10:20", "F401", "Error with row and column numbers")], - [ - ("10", "F401", "Multiple errors: the first error"), - ("20", "F402", "Multiple errors: the second error"), - ], - ], -) -def test_error_formatting(errors): - test_file = "project/test.py" - flake8_result = "[[bad]]\n" - for file_pos, code, text in errors: - flake8_result += f"{{test_dir}}/{test_file}:{file_pos}: [{code}] {text}\n" - flake8_result += "[[rst]]\n" - - runner = util.LinterRunner() - runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file]) - - run_result = runner.run_test([test_file], flake8_result=flake8_result) - - abs_test_file = runner.abs_source_file_path(test_file) - file_report = run_result.report_data["report"][abs_test_file] - expected_message_lines = [] - for file_pos, code, text in errors: - if ":" in file_pos: - row, col = file_pos.split(":") - col_with_sep = col + ":" - else: - row = file_pos - col_with_sep = "" - line = f"[[unimp]]{abs_test_file}[[rst]]:[[alt2]]{row}[[rst]]:[[alt2]]{col_with_sep}[[rst]] [[[alt1]]{code}[[rst]]] [[bad]]{text}[[rst]]" - expected_message_lines.append(line) - - assert file_report["status"] == "FAIL" - assert file_report["message"] == "\n".join(expected_message_lines) - assert file_report["elapsed"] >= 0.0 - - -def test_fail_on_wrong_message(): - test_file = "project/test.py" - flake8_result = """ - [[bad]] - Unexpected error message - [[rst]] - """ - - runner = util.LinterRunner() - runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file]) - - run_result = runner.run_test([test_file], flake8_result=flake8_result) - - assert run_result.linter_run_result.returncode != 0 - assert "Cannot parse flake8 output line" in run_result.linter_run_result.stderr diff --git a/tools/flake8_linter/bin/tests/util.py b/tools/flake8_linter/bin/tests/util.py deleted file mode 100644 index be3912e2bc..0000000000 --- a/tools/flake8_linter/bin/tests/util.py +++ /dev/null @@ -1,151 +0,0 @@ -import json -import logging -import mergedeep -import os -import shutil -import subprocess -import tempfile -from configparser import ConfigParser -from dataclasses import dataclass -from textwrap import dedent - -from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE -from yatest.common import work_path, binary_path - -# Config paths to reuse in different tests (just for convenience). This is not mandatory config paths. -FLAKE8_CONFIG_FILE = "build/config/flake8.cfg" -MIGRATIONS_CONFIG_FILE = "build/config/migrations.yaml" -DEFAULT_CONFIGS = [FLAKE8_CONFIG_FILE, MIGRATIONS_CONFIG_FILE] - -# Pass test parameters to flake8 stub via env variable -STUB_CONFIG_ENV_VAR_NAME = "_FLAKE8_STUB_CONFIG" - -logger = logging.getLogger(__name__) - - -@dataclass -class Flake8Launch: - flake8_bin: str - rel_file_paths: list[str] - config_data: str - - -@dataclass -class RunTestResult: - flake8_launches: list[Flake8Launch] - linter_run_result: subprocess.CompletedProcess - report_data: dict - - -class LinterRunner: - def __init__(self, lint_name: str = "py3_flake8"): - self._lint_name = lint_name - self._source_root = tempfile.mkdtemp(prefix="source_root", dir=work_path()) - self._work_root = tempfile.mkdtemp(prefix="work_root", dir=work_path()) - self._params_file = os.path.join(self._work_root, "params.json") - self._report_file = os.path.join(self._work_root, "report.json") - self._launch_report_file = os.path.join(self._work_root, "launches.json") - self._stub_config_file = os.path.join(self._work_root, "stub_config.json") - self._linter_path = binary_path("tools/flake8_linter/bin/flake8_linter") - self._global_resources = self._prepare_global_resources() - - def _prepare_global_resources(self): - global_resource_root = tempfile.mkdtemp(prefix="global_resources", dir=work_path()) - py2_stub_path = os.path.join(global_resource_root, "py2") - py3_stub_path = os.path.join(global_resource_root, "py3") - stub_path = binary_path("tools/flake8_linter/bin/tests/stub") - shutil.copytree(stub_path, py2_stub_path, copy_function=os.link) - shutil.copytree(stub_path, py3_stub_path, copy_function=os.link) - return { - FLAKE8_PY2_RESOURCE: py2_stub_path, - FLAKE8_PY3_RESOURCE: py3_stub_path, - } - - def create_source_file(self, rel_file_path: str, data: str): - abs_file_path = os.path.join(self._source_root, rel_file_path) - os.makedirs(os.path.dirname(abs_file_path), exist_ok=True) - with open(abs_file_path, "w") as f: - f.write(data) - - def create_source_tree(self, rel_file_paths: list[str]): - for rel_file_path in rel_file_paths: - self.create_source_file(rel_file_path, "") - - def abs_source_file_path(self, rel_file_path: str): - return os.path.join(self._source_root, rel_file_path) - - def run_test( - self, - file_rel_paths: list[str], - config_rel_paths: list[str] = DEFAULT_CONFIGS, - flake8_result: str = "", - custom_params: dict = {}, - env: dict[str, str] = {}, - ) -> RunTestResult: - self._prepare_params(config_rel_paths, file_rel_paths, custom_params) - stub_config = { - "output": dedent(flake8_result), - "report_file": self._launch_report_file, - } - stub_env = { - STUB_CONFIG_ENV_VAR_NAME: self._stub_config_file, - } - run_env = mergedeep.merge({}, env, stub_env) - with open(self._stub_config_file, "w") as f: - json.dump(stub_config, f) - linter_run_result = subprocess.run( - [self._linter_path, "--params", self._params_file], - encoding="utf-8", - capture_output=True, - check=False, - env=run_env, - ) - logger.debug("Linter run result: %s", str(linter_run_result)) - - if os.path.exists(self._report_file): - with open(self._report_file) as f: - report_data = json.load(f) - else: - report_data = None - - return RunTestResult(self._read_launches(), linter_run_result, report_data) - - def flake8_path(self, global_resource_var_name): - return self._global_resources[global_resource_var_name] - - def _prepare_params(self, config_rel_paths: list[str], file_rel_paths: list[str], custom_params: dict): - params = { - "source_root": self._source_root, - "project_path": "", - "output_path": "", - "lint_name": self._lint_name, - "depends": {}, - "global_resources": self._global_resources, - "configs": self._mk_source_abs_path(config_rel_paths), - "report_file": self._report_file, - "files": self._mk_source_abs_path(file_rel_paths), - } - mergedeep.merge(params, custom_params) - with open(self._params_file, "w") as f: - json.dump(params, f) - - def _mk_source_abs_path(self, paths): - return [self.abs_source_file_path(p) for p in paths] - - def _read_launches(self): - launches = [] - if os.path.exists(self._launch_report_file): - with open(self._launch_report_file) as f: - for line in f: - logger.debug("Launch report line: %s", line) - launch = json.loads(line) - launches.append(Flake8Launch(**launch)) - else: - logger.debug("Launch report file not found: %s", self._launch_report_file) - return launches - - -def assert_configs(got: ConfigParser, expected: ConfigParser): - got_dict = dict(got["flake8"].items()) - expected_dict = dict(expected["flake8"].items()) - assert got_dict == expected_dict diff --git a/tools/flake8_linter/bin/tests/ya.make b/tools/flake8_linter/bin/tests/ya.make deleted file mode 100644 index 533a02a1f1..0000000000 --- a/tools/flake8_linter/bin/tests/ya.make +++ /dev/null @@ -1,29 +0,0 @@ -PY3TEST() - -STYLE_PYTHON() - -TEST_SRCS( - test_flake8_ver.py - test_migrations.py - test_noqa.py - test_report.py - util.py -) - -PEERDIR( - build/plugins/lib - contrib/python/mergedeep - devtools/ya/test/tests/lib/common - library/python/testing/custom_linter_util -) - -DEPENDS( - tools/flake8_linter/bin - tools/flake8_linter/bin/tests/stub -) - -END() - -RECURSE( - stub -) diff --git a/tools/flake8_linter/bin/ya.make b/tools/flake8_linter/bin/ya.make deleted file mode 100644 index 197c4b9404..0000000000 --- a/tools/flake8_linter/bin/ya.make +++ /dev/null @@ -1,20 +0,0 @@ -PY3_PROGRAM(flake8_linter) - -STYLE_PYTHON() - -PEERDIR( - build/plugins/lib/test_const - devtools/ya/test/programs/test_tool/lib/migrations_config - devtools/ya/yalibrary/term - library/python/testing/custom_linter_util -) - -SRCDIR( - tools/flake8_linter/bin -) - -PY_SRCS( - __main__.py -) - -END() diff --git a/tools/flake8_linter/ya.make b/tools/flake8_linter/ya.make deleted file mode 100644 index eac0ce5f21..0000000000 --- a/tools/flake8_linter/ya.make +++ /dev/null @@ -1,15 +0,0 @@ -IF (USE_PREBUILT_TOOLS OR OPENSOURCE) - INCLUDE(${ARCADIA_ROOT}/build/prebuilt/tools/flake8_linter/ya.make.prebuilt) -ENDIF() - -IF (NOT PREBUILT) - INCLUDE(${ARCADIA_ROOT}/tools/flake8_linter/bin/ya.make) -ENDIF() - -RECURSE( - bin -) - -RECURSE_FOR_TESTS( - bin/tests -) |