aboutsummaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
authorprettyboy <prettyboy@yandex-team.com>2023-09-08 10:08:57 +0300
committerprettyboy <prettyboy@yandex-team.com>2023-09-08 10:39:21 +0300
commit329f805999a3b41e406959a17cf35ab193ef05a5 (patch)
tree9a3059df4d544b6c9d6f474344e52f65bd13b4c1 /tools
parent02ea6261088be81bbc455933cecf8b41726946c1 (diff)
downloadydb-329f805999a3b41e406959a17cf35ab193ef05a5.tar.gz
Revert commit rXXXXXX,[build/plugins/ytest] Allow prebuilt linters for opensource
Diffstat (limited to 'tools')
-rw-r--r--tools/black_linter/ya.make11
-rw-r--r--tools/flake8_linter/bin/__main__.py190
-rw-r--r--tools/flake8_linter/bin/tests/stub/__main__.py51
-rw-r--r--tools/flake8_linter/bin/tests/stub/ya.make9
-rw-r--r--tools/flake8_linter/bin/tests/test_flake8_ver.py30
-rw-r--r--tools/flake8_linter/bin/tests/test_migrations.py164
-rw-r--r--tools/flake8_linter/bin/tests/test_noqa.py73
-rw-r--r--tools/flake8_linter/bin/tests/test_report.py101
-rw-r--r--tools/flake8_linter/bin/tests/util.py151
-rw-r--r--tools/flake8_linter/bin/tests/ya.make29
-rw-r--r--tools/flake8_linter/bin/ya.make20
-rw-r--r--tools/flake8_linter/ya.make15
12 files changed, 0 insertions, 844 deletions
diff --git a/tools/black_linter/ya.make b/tools/black_linter/ya.make
deleted file mode 100644
index b5607dc159..0000000000
--- a/tools/black_linter/ya.make
+++ /dev/null
@@ -1,11 +0,0 @@
-IF (USE_PREBUILT_TOOLS OR OPENSOURCE)
- INCLUDE(${ARCADIA_ROOT}/build/prebuilt/tools/black_linter/ya.make.prebuilt)
-ENDIF()
-
-IF (NOT PREBUILT)
- INCLUDE(${ARCADIA_ROOT}/tools/black_linter/bin/ya.make)
-ENDIF()
-
-RECURSE(
- bin
-)
diff --git a/tools/flake8_linter/bin/__main__.py b/tools/flake8_linter/bin/__main__.py
deleted file mode 100644
index 77a8edc881..0000000000
--- a/tools/flake8_linter/bin/__main__.py
+++ /dev/null
@@ -1,190 +0,0 @@
-import configparser
-import hashlib
-import itertools
-import io
-import logging
-import os
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
-from collections import defaultdict
-from typing import Generator
-
-from devtools.ya.test.programs.test_tool.lib.migrations_config import load_yaml_config, MigrationsConfig
-from library.python.testing.custom_linter_util import linter_params, reporter
-from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE
-
-logger = logging.getLogger(__name__)
-
-ALLOWED_IGNORES = {"F401"}
-# Supports both default and pylint formats
-FLAKE_LINE_RE = r"^(.*?):(\d+):(\d+:)? \[(\w+\d+)\] (.*)"
-FLAKE8_CONFIG_INDEX = 0
-MIGRATIONS_CONFIG_INDEX = 1
-
-
-def get_flake8_bin(params) -> str:
- if params.lint_name == "py2_flake8":
- flake8_root = params.global_resources[FLAKE8_PY2_RESOURCE]
- elif params.lint_name == "py3_flake8":
- flake8_root = params.global_resources[FLAKE8_PY3_RESOURCE]
- else:
- raise RuntimeError("Unexpected lint name: {}".format(params.lint_name))
- return os.path.join(flake8_root, "flake8")
-
-
-def get_migrations_config(params) -> MigrationsConfig:
- if params.extra_params.get("DISABLE_FLAKE8_MIGRATIONS", "no") == "yes":
- return MigrationsConfig()
- config_path = os.getenv("_YA_TEST_FLAKE8_CONFIG", params.configs[MIGRATIONS_CONFIG_INDEX])
- if not config_path:
- return MigrationsConfig()
- else:
- logger.debug("Loading flake8 migrations: %s", config_path)
- migrations = load_yaml_config(config_path)
- logger.debug("Building migration config")
- return MigrationsConfig(migrations)
-
-
-def get_flake8_config(
- flake8_config: str, migrations_config: MigrationsConfig, source_root: str, file_path: str
-) -> str | None:
- arc_rel_file_path = os.path.relpath(file_path, source_root)
- if migrations_config.is_skipped(arc_rel_file_path):
- return None
- exceptions = migrations_config.get_exceptions(arc_rel_file_path)
- if exceptions:
- logger.info("Ignore flake8 exceptions %s for file %s", str(list(exceptions)), arc_rel_file_path)
-
- if os.path.basename(file_path) == "__init__.py":
- exceptions |= get_noqa_exceptions(file_path)
-
- if exceptions:
- new_config = configparser.ConfigParser()
- new_config.read(flake8_config) # https://bugs.python.org/issue16058 Why don't use deepcopy
- new_config["flake8"]["ignore"] += "\n" + "\n".join(x + "," for x in sorted(exceptions))
-
- config_stream = io.StringIO()
- new_config.write(config_stream)
- config_hash = hashlib.md5(config_stream.getvalue().encode()).hexdigest()
- config_path = config_hash + ".config"
- if not os.path.exists(config_path):
- with open(config_path, "w") as f:
- f.write(config_stream.getvalue())
- return config_path
- else:
- return flake8_config
-
-
-def get_noqa_exceptions(file_path: str) -> set:
- additional_exceptions = get_file_ignores(file_path)
- validate_exceptions(additional_exceptions)
- return additional_exceptions & ALLOWED_IGNORES
-
-
-def get_file_ignores(file_path: str) -> set:
- file_ignore_regex = re.compile(r"#\s*flake8\s+noqa:\s*(.*)")
- with open(file_path) as afile:
- # looking for ignores only in the first 3 lines
- for line in itertools.islice(afile, 3):
- if match := file_ignore_regex.search(line):
- ignores = match.group(1).strip()
- if ignores:
- ignores = re.split(r"\s*,\s*", ignores)
- return set(ignores)
- return set()
-
-
-def validate_exceptions(exceptions: set) -> None:
- if exceptions - ALLOWED_IGNORES:
- logger.error(
- "Disabling %s checks. Only %s can be suppressed in the __init__.py files using # flake8 noqa",
- str(list(exceptions - ALLOWED_IGNORES)),
- str(list(ALLOWED_IGNORES)),
- )
-
-
-def run_flake8_for_dir(flake8_bin: str, source_root: str, config: str, check_files: list[str]) -> dict[str, list[str]]:
- with tempfile.TemporaryDirectory() as temp_dir:
- logger.debug("flake8 temp dir: %s", temp_dir)
- for f in check_files:
- copy_file_path = os.path.join(temp_dir, os.path.relpath(f, source_root))
- os.makedirs(os.path.dirname(copy_file_path), exist_ok=True)
- shutil.copyfile(f, copy_file_path)
- flake8_res = run_flake8(flake8_bin, temp_dir, config)
- return get_flake8_results(flake8_res, source_root, temp_dir)
-
-
-def run_flake8(flake8_bin: str, dir_path: str, config: str) -> list[str]:
- cmd = [flake8_bin, dir_path, config]
- res = subprocess.run(cmd, capture_output=True, encoding="utf8", errors="replace")
- if res.stderr:
- logger.debug("flake8 stderr: %s", res.stderr)
- return res.stdout.split("\n") if res.returncode else []
-
-
-def get_flake8_results(flake8_res: list[str], source_root: str, temp_dir: str) -> dict[str, list[str]]:
- flake8_errors_map = defaultdict(list)
- for line in iterate_over_results(flake8_res):
- match = re.match(FLAKE_LINE_RE, line)
- if not match:
- raise RuntimeError("Cannot parse flake8 output line: '{}'".format(line))
- file_path, row, col_with_sep, code, text = match.groups()
- file_path = file_path.replace(temp_dir, source_root)
- if col_with_sep is None:
- col_with_sep = ""
- colorized_line = f"[[unimp]]{file_path}[[rst]]:[[alt2]]{row}[[rst]]:[[alt2]]{col_with_sep}[[rst]] [[[alt1]]{code}[[rst]]] [[bad]]{text}[[rst]]"
- flake8_errors_map[file_path].append(colorized_line)
- return flake8_errors_map
-
-
-def iterate_over_results(flake8_res: list[str]) -> Generator[str, None, None]:
- to_skip = {"[[bad]]", "[[rst]]"}
- for line in flake8_res:
- if line and line not in to_skip:
- yield line
-
-
-def main():
- params = linter_params.get_params()
- logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format="%(asctime)s: %(levelname)s: %(message)s")
-
- flake8_bin = get_flake8_bin(params)
- flake8_config = params.configs[FLAKE8_CONFIG_INDEX]
- migrations_config = get_migrations_config(params)
- source_root = params.source_root
-
- logger.debug("Constructing flake8 config")
- config_map = defaultdict(list)
- report = reporter.LintReport()
-
- skipped_files = set()
- for file_path in params.files:
- config_path = get_flake8_config(flake8_config, migrations_config, source_root, file_path)
- if config_path:
- config_map[config_path].append(file_path)
- else:
- skipped_files.add(file_path)
-
- logger.debug("Configuration:\n%s", str(config_map))
-
- flake8_errors_map = {}
- for config_path, check_files in config_map.items():
- flake8_errors_map.update(run_flake8_for_dir(flake8_bin, source_root, config_path, check_files))
-
- report = reporter.LintReport()
- for file_path in params.files:
- if file_path in skipped_files:
- report.add(file_path, reporter.LintStatus.SKIPPED, "Skipped by config")
- elif file_path in flake8_errors_map:
- message = "\n".join(flake8_errors_map[file_path])
- report.add(file_path, reporter.LintStatus.FAIL, message)
- else:
- report.add(file_path, reporter.LintStatus.GOOD)
- report.dump(params.report_file)
-
-
-if __name__ == "__main__":
- main()
diff --git a/tools/flake8_linter/bin/tests/stub/__main__.py b/tools/flake8_linter/bin/tests/stub/__main__.py
deleted file mode 100644
index 3bbfcf424b..0000000000
--- a/tools/flake8_linter/bin/tests/stub/__main__.py
+++ /dev/null
@@ -1,51 +0,0 @@
-"""
-flake8 emulator. Does the following:
-- read config file (the name is specified in _FLAKE8_STUB_CONFIG env variable)
-- gather launch info and put it into the report file
-- print test data to stdout
-"""
-
-import json
-import os
-import sys
-
-
-STUB_CONFIG_ENV_VAR_NAME = "_FLAKE8_STUB_CONFIG"
-
-
-def main():
- flake8_bin, test_dir, flake8_config = sys.argv
- stub_config_file = os.getenv(STUB_CONFIG_ENV_VAR_NAME)
- with open(stub_config_file) as f:
- stub_config = json.load(f)
-
- stub_output = stub_config["output"]
- launch_report_file = stub_config["report_file"]
-
- launch_report = get_launch_report(flake8_bin, test_dir, flake8_config)
- with open(launch_report_file, "a") as f:
- json.dump(launch_report, f)
- f.write("\n")
-
- if stub_output:
- sys.stdout.write(stub_output.format(test_dir=test_dir))
- return 1
- else:
- return 0
-
-
-def get_launch_report(flake8_bin, test_dir, flake8_config):
- rel_file_paths = []
- for root, _, files in os.walk(test_dir):
- rel_file_paths += [os.path.relpath(os.path.join(root, f), test_dir) for f in files]
- with open(flake8_config) as f:
- config_data = f.read()
- return {
- "flake8_bin": flake8_bin,
- "rel_file_paths": rel_file_paths,
- "config_data": config_data,
- }
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/tools/flake8_linter/bin/tests/stub/ya.make b/tools/flake8_linter/bin/tests/stub/ya.make
deleted file mode 100644
index 8c24917942..0000000000
--- a/tools/flake8_linter/bin/tests/stub/ya.make
+++ /dev/null
@@ -1,9 +0,0 @@
-PY3_PROGRAM(flake8)
-
-STYLE_PYTHON()
-
-PY_SRCS(
- __main__.py
-)
-
-END()
diff --git a/tools/flake8_linter/bin/tests/test_flake8_ver.py b/tools/flake8_linter/bin/tests/test_flake8_ver.py
deleted file mode 100644
index d2760a8c7d..0000000000
--- a/tools/flake8_linter/bin/tests/test_flake8_ver.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import os
-import pytest
-
-from . import util
-from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE
-
-
-@pytest.mark.parametrize(
- "lint_name, global_resource_var_name",
- [
- ("py2_flake8", FLAKE8_PY2_RESOURCE),
- ("py3_flake8", FLAKE8_PY3_RESOURCE),
- ],
-)
-def test_flake8_version(lint_name, global_resource_var_name):
- test_file = "project/test.py"
- runner = util.LinterRunner(lint_name)
- runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
- run_result = runner.run_test([test_file])
- expected_flake8_bin = os.path.join(runner.flake8_path(global_resource_var_name), "flake8")
- assert run_result.flake8_launches[0].flake8_bin == expected_flake8_bin
-
-
-def test_raise_on_incorrect_lint_name():
- test_file = "project/test.py"
- runner = util.LinterRunner("strange_lint_name")
- runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
- run_result = runner.run_test([test_file])
- assert run_result.linter_run_result.returncode != 0
- assert "Unexpected lint name" in run_result.linter_run_result.stderr
diff --git a/tools/flake8_linter/bin/tests/test_migrations.py b/tools/flake8_linter/bin/tests/test_migrations.py
deleted file mode 100644
index 1e77d265be..0000000000
--- a/tools/flake8_linter/bin/tests/test_migrations.py
+++ /dev/null
@@ -1,164 +0,0 @@
-import pytest
-from configparser import ConfigParser
-
-from . import util
-
-FLAKE8_CONFIG_DATA = """
- [flake8]
- select = E, W, F
- ignore =
- E122,
- E743,
- F403,
- W605,
-"""
-MIGRATION_CONFIG_DATA = """
- flake8:
- F401:
- ignore:
- - F401
- prefixes:
- - project401
- SKIP:
- ignore:
- - "*"
- prefixes:
- - project_skip
-"""
-FILE_WITHOUT_EXCEPTIONS = "project/file1.py"
-FILE_IGNORE_F401 = "project401/file2.py"
-FILE_SKIPPED = "project_skip/file3.py"
-
-
-@pytest.mark.parametrize(
- "test_file, added_ignore, disable_migrations",
- [
- (FILE_WITHOUT_EXCEPTIONS, "", False),
- (FILE_IGNORE_F401, "\nF401,", False),
- (FILE_IGNORE_F401, "", True),
- ],
-)
-def test_ignore(test_file, added_ignore, disable_migrations):
- test_files = [test_file]
-
- runner = util.LinterRunner()
- runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
- runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA)
- runner.create_source_tree(test_files)
-
- disable_migrations_params = {"extra_params": {"DISABLE_FLAKE8_MIGRATIONS": "yes"}} if disable_migrations else {}
- run_result = runner.run_test(test_files, custom_params=disable_migrations_params)
-
- assert len(run_result.flake8_launches) == 1
-
- launch = run_result.flake8_launches[0]
- got_config = ConfigParser()
- got_config.read_string(launch.config_data)
- expected_config = ConfigParser()
- expected_config.read_string(FLAKE8_CONFIG_DATA)
- expected_config["flake8"]["ignore"] += added_ignore
-
- util.assert_configs(got_config, expected_config)
-
- assert launch.rel_file_paths == test_files
-
-
-def test_skipped():
- test_files = [FILE_SKIPPED]
-
- runner = util.LinterRunner()
- runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
- runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA)
- runner.create_source_tree(test_files)
-
- run_result = runner.run_test(test_files)
-
- assert len(run_result.flake8_launches) == 0
-
- abs_test_file_path = runner.abs_source_file_path(FILE_SKIPPED)
- assert run_result.report_data["report"][abs_test_file_path]["status"] == "SKIPPED"
-
-
-def test_group_files_by_config():
- test_files = [FILE_WITHOUT_EXCEPTIONS, FILE_IGNORE_F401]
-
- runner = util.LinterRunner()
- runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
- runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA)
- runner.create_source_tree(test_files)
-
- run_result = runner.run_test(test_files)
-
- assert len(run_result.flake8_launches) == 2
-
- for launch in run_result.flake8_launches:
- rel_file_paths = launch.rel_file_paths
- got_config = ConfigParser()
- got_config.read_string(launch.config_data)
- # Relaxed check if config is matched with a checked file
- # Thorough config check is done in test_ignore()
- if rel_file_paths == [FILE_WITHOUT_EXCEPTIONS]:
- assert "F401" not in got_config["flake8"]["ignore"]
- elif rel_file_paths == [FILE_IGNORE_F401]:
- assert "F401" in got_config["flake8"]["ignore"]
- else:
- pytest.fail("Unexpected file paths passed to flake8 binary: {}".format(rel_file_paths))
-
-
-@pytest.mark.parametrize(
- "migrations_file, expected_ignore",
- [
- (None, "F777"),
- ("", None),
- ("build/config/other_migration.yaml", "F888"),
- ],
-)
-def test_migration_file_from_env(migrations_file, expected_ignore):
- # Env var _YA_TEST_FLAKE8_CONFIG overrides file name from configs parameter. _YA_TEST_FLAKE8_CONFIG:
- # - is not defined - use migrations file from configs parameter
- # - is empty - don't use migrations file at all
- # - not empty - use variable value as migrations file name
- config_migrations = """
- flake8:
- ignore:
- ignore:
- - F777
- prefixes:
- - project
- """
- test_files = ["project/test.py"]
-
- runner = util.LinterRunner()
- runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
- runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, config_migrations)
- runner.create_source_tree(test_files)
- env = {}
- if migrations_file is not None:
- if migrations_file:
- env_var_migrations = """
- flake8:
- ignore:
- ignore:
- - F888
- prefixes:
- - project
- """
- runner.create_source_file(migrations_file, env_var_migrations)
- env_var_value = runner.abs_source_file_path(migrations_file)
- else:
- env_var_value = ""
- env["_YA_TEST_FLAKE8_CONFIG"] = env_var_value
-
- run_result = runner.run_test(test_files, env=env)
-
- assert len(run_result.flake8_launches) == 1
-
- launch = run_result.flake8_launches[0]
- got_config = ConfigParser()
- got_config.read_string(launch.config_data)
- ignores = got_config["flake8"]["ignore"]
- if expected_ignore:
- assert expected_ignore in ignores
- else:
- assert "F777" not in ignores
- assert "F888" not in ignores
diff --git a/tools/flake8_linter/bin/tests/test_noqa.py b/tools/flake8_linter/bin/tests/test_noqa.py
deleted file mode 100644
index 1f7d536fe7..0000000000
--- a/tools/flake8_linter/bin/tests/test_noqa.py
+++ /dev/null
@@ -1,73 +0,0 @@
-import io
-import pytest
-from configparser import ConfigParser
-
-from . import util
-
-FLAKE8_CONFIG_DATA = """
- [flake8]
- select = E, W, F
- ignore =
- E122,
-"""
-
-
-@pytest.mark.parametrize(
- "file_name, noqa_line_no, is_scanned",
- [
- ("__init__.py", 1, True),
- ("__init__.py", 3, True),
- ("__init__.py", 4, False),
- ("not_init.py", 1, False),
- ],
-)
-def test_scanned_line_count(file_name, noqa_line_no, is_scanned):
- test_file = "project/" + file_name
-
- test_file_data = io.StringIO()
- for lno in range(1, 10):
- test_file_data.write("pass")
- if lno == noqa_line_no:
- test_file_data.write(" # flake8 noqa: F401")
- test_file_data.write("\n")
-
- runner = util.LinterRunner()
- runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
- runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, "")
- runner.create_source_file(test_file, test_file_data.getvalue())
-
- run_result = runner.run_test([test_file])
-
- assert len(run_result.flake8_launches) == 1
-
- launch = run_result.flake8_launches[0]
- got_config = ConfigParser()
- got_config.read_string(launch.config_data)
- if is_scanned:
- assert "F401" in got_config["flake8"]["ignore"]
- else:
- assert "F401" not in got_config["flake8"]["ignore"]
-
-
-def test_not_F401():
- test_file = "project/__init__.py"
-
- test_file_data = """
- pass # flake8 noqa: F777, F401
- """
-
- runner = util.LinterRunner()
- runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
- runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, "")
- runner.create_source_file(test_file, test_file_data)
-
- run_result = runner.run_test([test_file])
-
- assert len(run_result.flake8_launches) == 1
-
- launch = run_result.flake8_launches[0]
- got_config = ConfigParser()
- got_config.read_string(launch.config_data)
- assert "F401" in got_config["flake8"]["ignore"]
- assert "F777" not in got_config["flake8"]["ignore"]
- assert "Disabling ['F777'] checks" in run_result.linter_run_result.stdout
diff --git a/tools/flake8_linter/bin/tests/test_report.py b/tools/flake8_linter/bin/tests/test_report.py
deleted file mode 100644
index 9304fe86b3..0000000000
--- a/tools/flake8_linter/bin/tests/test_report.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import pytest
-
-from . import util
-
-FLAKE8_CONFIG_DATA = """
- [flake8]
- select = E, W, F
- ignore =
- E122,
-"""
-
-
-def test_no_errors():
- test_file = "project/test.py"
- runner = util.LinterRunner()
- runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
-
- run_result = runner.run_test([test_file])
-
- abs_test_file = runner.abs_source_file_path(test_file)
- file_report = run_result.report_data["report"][abs_test_file]
- assert file_report["status"] == "GOOD"
- assert file_report["message"] == ""
- assert file_report["elapsed"] >= 0.0
-
-
-def test_skip_markup():
- test_file = "project/test.py"
- flake8_result = """
- [[bad]]
- [[rst]]
- """
-
- runner = util.LinterRunner()
- runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
-
- run_result = runner.run_test([test_file], flake8_result=flake8_result)
-
- abs_test_file = runner.abs_source_file_path(test_file)
- file_report = run_result.report_data["report"][abs_test_file]
- assert file_report["status"] == "GOOD"
- assert file_report["message"] == ""
- assert file_report["elapsed"] >= 0.0
-
-
-@pytest.mark.parametrize(
- "errors",
- [
- [("10", "F401", "Error with row number only")],
- [("10:20", "F401", "Error with row and column numbers")],
- [
- ("10", "F401", "Multiple errors: the first error"),
- ("20", "F402", "Multiple errors: the second error"),
- ],
- ],
-)
-def test_error_formatting(errors):
- test_file = "project/test.py"
- flake8_result = "[[bad]]\n"
- for file_pos, code, text in errors:
- flake8_result += f"{{test_dir}}/{test_file}:{file_pos}: [{code}] {text}\n"
- flake8_result += "[[rst]]\n"
-
- runner = util.LinterRunner()
- runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
-
- run_result = runner.run_test([test_file], flake8_result=flake8_result)
-
- abs_test_file = runner.abs_source_file_path(test_file)
- file_report = run_result.report_data["report"][abs_test_file]
- expected_message_lines = []
- for file_pos, code, text in errors:
- if ":" in file_pos:
- row, col = file_pos.split(":")
- col_with_sep = col + ":"
- else:
- row = file_pos
- col_with_sep = ""
- line = f"[[unimp]]{abs_test_file}[[rst]]:[[alt2]]{row}[[rst]]:[[alt2]]{col_with_sep}[[rst]] [[[alt1]]{code}[[rst]]] [[bad]]{text}[[rst]]"
- expected_message_lines.append(line)
-
- assert file_report["status"] == "FAIL"
- assert file_report["message"] == "\n".join(expected_message_lines)
- assert file_report["elapsed"] >= 0.0
-
-
-def test_fail_on_wrong_message():
- test_file = "project/test.py"
- flake8_result = """
- [[bad]]
- Unexpected error message
- [[rst]]
- """
-
- runner = util.LinterRunner()
- runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
-
- run_result = runner.run_test([test_file], flake8_result=flake8_result)
-
- assert run_result.linter_run_result.returncode != 0
- assert "Cannot parse flake8 output line" in run_result.linter_run_result.stderr
diff --git a/tools/flake8_linter/bin/tests/util.py b/tools/flake8_linter/bin/tests/util.py
deleted file mode 100644
index be3912e2bc..0000000000
--- a/tools/flake8_linter/bin/tests/util.py
+++ /dev/null
@@ -1,151 +0,0 @@
-import json
-import logging
-import mergedeep
-import os
-import shutil
-import subprocess
-import tempfile
-from configparser import ConfigParser
-from dataclasses import dataclass
-from textwrap import dedent
-
-from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE
-from yatest.common import work_path, binary_path
-
-# Config paths to reuse in different tests (just for convenience). This is not mandatory config paths.
-FLAKE8_CONFIG_FILE = "build/config/flake8.cfg"
-MIGRATIONS_CONFIG_FILE = "build/config/migrations.yaml"
-DEFAULT_CONFIGS = [FLAKE8_CONFIG_FILE, MIGRATIONS_CONFIG_FILE]
-
-# Pass test parameters to flake8 stub via env variable
-STUB_CONFIG_ENV_VAR_NAME = "_FLAKE8_STUB_CONFIG"
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class Flake8Launch:
- flake8_bin: str
- rel_file_paths: list[str]
- config_data: str
-
-
-@dataclass
-class RunTestResult:
- flake8_launches: list[Flake8Launch]
- linter_run_result: subprocess.CompletedProcess
- report_data: dict
-
-
-class LinterRunner:
- def __init__(self, lint_name: str = "py3_flake8"):
- self._lint_name = lint_name
- self._source_root = tempfile.mkdtemp(prefix="source_root", dir=work_path())
- self._work_root = tempfile.mkdtemp(prefix="work_root", dir=work_path())
- self._params_file = os.path.join(self._work_root, "params.json")
- self._report_file = os.path.join(self._work_root, "report.json")
- self._launch_report_file = os.path.join(self._work_root, "launches.json")
- self._stub_config_file = os.path.join(self._work_root, "stub_config.json")
- self._linter_path = binary_path("tools/flake8_linter/bin/flake8_linter")
- self._global_resources = self._prepare_global_resources()
-
- def _prepare_global_resources(self):
- global_resource_root = tempfile.mkdtemp(prefix="global_resources", dir=work_path())
- py2_stub_path = os.path.join(global_resource_root, "py2")
- py3_stub_path = os.path.join(global_resource_root, "py3")
- stub_path = binary_path("tools/flake8_linter/bin/tests/stub")
- shutil.copytree(stub_path, py2_stub_path, copy_function=os.link)
- shutil.copytree(stub_path, py3_stub_path, copy_function=os.link)
- return {
- FLAKE8_PY2_RESOURCE: py2_stub_path,
- FLAKE8_PY3_RESOURCE: py3_stub_path,
- }
-
- def create_source_file(self, rel_file_path: str, data: str):
- abs_file_path = os.path.join(self._source_root, rel_file_path)
- os.makedirs(os.path.dirname(abs_file_path), exist_ok=True)
- with open(abs_file_path, "w") as f:
- f.write(data)
-
- def create_source_tree(self, rel_file_paths: list[str]):
- for rel_file_path in rel_file_paths:
- self.create_source_file(rel_file_path, "")
-
- def abs_source_file_path(self, rel_file_path: str):
- return os.path.join(self._source_root, rel_file_path)
-
- def run_test(
- self,
- file_rel_paths: list[str],
- config_rel_paths: list[str] = DEFAULT_CONFIGS,
- flake8_result: str = "",
- custom_params: dict = {},
- env: dict[str, str] = {},
- ) -> RunTestResult:
- self._prepare_params(config_rel_paths, file_rel_paths, custom_params)
- stub_config = {
- "output": dedent(flake8_result),
- "report_file": self._launch_report_file,
- }
- stub_env = {
- STUB_CONFIG_ENV_VAR_NAME: self._stub_config_file,
- }
- run_env = mergedeep.merge({}, env, stub_env)
- with open(self._stub_config_file, "w") as f:
- json.dump(stub_config, f)
- linter_run_result = subprocess.run(
- [self._linter_path, "--params", self._params_file],
- encoding="utf-8",
- capture_output=True,
- check=False,
- env=run_env,
- )
- logger.debug("Linter run result: %s", str(linter_run_result))
-
- if os.path.exists(self._report_file):
- with open(self._report_file) as f:
- report_data = json.load(f)
- else:
- report_data = None
-
- return RunTestResult(self._read_launches(), linter_run_result, report_data)
-
- def flake8_path(self, global_resource_var_name):
- return self._global_resources[global_resource_var_name]
-
- def _prepare_params(self, config_rel_paths: list[str], file_rel_paths: list[str], custom_params: dict):
- params = {
- "source_root": self._source_root,
- "project_path": "",
- "output_path": "",
- "lint_name": self._lint_name,
- "depends": {},
- "global_resources": self._global_resources,
- "configs": self._mk_source_abs_path(config_rel_paths),
- "report_file": self._report_file,
- "files": self._mk_source_abs_path(file_rel_paths),
- }
- mergedeep.merge(params, custom_params)
- with open(self._params_file, "w") as f:
- json.dump(params, f)
-
- def _mk_source_abs_path(self, paths):
- return [self.abs_source_file_path(p) for p in paths]
-
- def _read_launches(self):
- launches = []
- if os.path.exists(self._launch_report_file):
- with open(self._launch_report_file) as f:
- for line in f:
- logger.debug("Launch report line: %s", line)
- launch = json.loads(line)
- launches.append(Flake8Launch(**launch))
- else:
- logger.debug("Launch report file not found: %s", self._launch_report_file)
- return launches
-
-
-def assert_configs(got: ConfigParser, expected: ConfigParser):
- got_dict = dict(got["flake8"].items())
- expected_dict = dict(expected["flake8"].items())
- assert got_dict == expected_dict
diff --git a/tools/flake8_linter/bin/tests/ya.make b/tools/flake8_linter/bin/tests/ya.make
deleted file mode 100644
index 533a02a1f1..0000000000
--- a/tools/flake8_linter/bin/tests/ya.make
+++ /dev/null
@@ -1,29 +0,0 @@
-PY3TEST()
-
-STYLE_PYTHON()
-
-TEST_SRCS(
- test_flake8_ver.py
- test_migrations.py
- test_noqa.py
- test_report.py
- util.py
-)
-
-PEERDIR(
- build/plugins/lib
- contrib/python/mergedeep
- devtools/ya/test/tests/lib/common
- library/python/testing/custom_linter_util
-)
-
-DEPENDS(
- tools/flake8_linter/bin
- tools/flake8_linter/bin/tests/stub
-)
-
-END()
-
-RECURSE(
- stub
-)
diff --git a/tools/flake8_linter/bin/ya.make b/tools/flake8_linter/bin/ya.make
deleted file mode 100644
index 197c4b9404..0000000000
--- a/tools/flake8_linter/bin/ya.make
+++ /dev/null
@@ -1,20 +0,0 @@
-PY3_PROGRAM(flake8_linter)
-
-STYLE_PYTHON()
-
-PEERDIR(
- build/plugins/lib/test_const
- devtools/ya/test/programs/test_tool/lib/migrations_config
- devtools/ya/yalibrary/term
- library/python/testing/custom_linter_util
-)
-
-SRCDIR(
- tools/flake8_linter/bin
-)
-
-PY_SRCS(
- __main__.py
-)
-
-END()
diff --git a/tools/flake8_linter/ya.make b/tools/flake8_linter/ya.make
deleted file mode 100644
index eac0ce5f21..0000000000
--- a/tools/flake8_linter/ya.make
+++ /dev/null
@@ -1,15 +0,0 @@
-IF (USE_PREBUILT_TOOLS OR OPENSOURCE)
- INCLUDE(${ARCADIA_ROOT}/build/prebuilt/tools/flake8_linter/ya.make.prebuilt)
-ENDIF()
-
-IF (NOT PREBUILT)
- INCLUDE(${ARCADIA_ROOT}/tools/flake8_linter/bin/ya.make)
-ENDIF()
-
-RECURSE(
- bin
-)
-
-RECURSE_FOR_TESTS(
- bin/tests
-)