[build/plugins/ytest] Allow prebuilt linters for opensource

Без этого, ydb или не сможет запускать flake8 с помощью ya make.
Или к ним поедет сборка flake8.
Возможно последнее и не так плохо, но сейчас предлагается пока так

12 files changed, 844 insertions, 0 deletions

diff --git a/tools/black_linter/ya.make b/tools/black_linter/ya.make
new file mode 100644
index 00000000000..b5607dc1593
--- /dev/null
+++ b/tools/black_linter/ya.make
@@ -0,0 +1,11 @@
+IF (USE_PREBUILT_TOOLS OR OPENSOURCE)
+    INCLUDE(${ARCADIA_ROOT}/build/prebuilt/tools/black_linter/ya.make.prebuilt)
+ENDIF()
+
+IF (NOT PREBUILT)
+    INCLUDE(${ARCADIA_ROOT}/tools/black_linter/bin/ya.make)
+ENDIF()
+
+RECURSE(
+    bin
+)
diff --git a/tools/flake8_linter/bin/__main__.py b/tools/flake8_linter/bin/__main__.py
new file mode 100644
index 00000000000..77a8edc881b
--- /dev/null
+++ b/tools/flake8_linter/bin/__main__.py
@@ -0,0 +1,190 @@
+import configparser
+import hashlib
+import itertools
+import io
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from collections import defaultdict
+from typing import Generator
+
+from devtools.ya.test.programs.test_tool.lib.migrations_config import load_yaml_config, MigrationsConfig
+from library.python.testing.custom_linter_util import linter_params, reporter
+from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE
+
+logger = logging.getLogger(__name__)
+
+ALLOWED_IGNORES = {"F401"}
+# Supports both default and pylint formats
+FLAKE_LINE_RE = r"^(.*?):(\d+):(\d+:)? \[(\w+\d+)\] (.*)"
+FLAKE8_CONFIG_INDEX = 0
+MIGRATIONS_CONFIG_INDEX = 1
+
+
+def get_flake8_bin(params) -> str:
+    if params.lint_name == "py2_flake8":
+        flake8_root = params.global_resources[FLAKE8_PY2_RESOURCE]
+    elif params.lint_name == "py3_flake8":
+        flake8_root = params.global_resources[FLAKE8_PY3_RESOURCE]
+    else:
+        raise RuntimeError("Unexpected lint name: {}".format(params.lint_name))
+    return os.path.join(flake8_root, "flake8")
+
+
+def get_migrations_config(params) -> MigrationsConfig:
+    if params.extra_params.get("DISABLE_FLAKE8_MIGRATIONS", "no") == "yes":
+        return MigrationsConfig()
+    config_path = os.getenv("_YA_TEST_FLAKE8_CONFIG", params.configs[MIGRATIONS_CONFIG_INDEX])
+    if not config_path:
+        return MigrationsConfig()
+    else:
+        logger.debug("Loading flake8 migrations: %s", config_path)
+        migrations = load_yaml_config(config_path)
+        logger.debug("Building migration config")
+        return MigrationsConfig(migrations)
+
+
+def get_flake8_config(
+    flake8_config: str, migrations_config: MigrationsConfig, source_root: str, file_path: str
+) -> str | None:
+    arc_rel_file_path = os.path.relpath(file_path, source_root)
+    if migrations_config.is_skipped(arc_rel_file_path):
+        return None
+    exceptions = migrations_config.get_exceptions(arc_rel_file_path)
+    if exceptions:
+        logger.info("Ignore flake8 exceptions %s for file %s", str(list(exceptions)), arc_rel_file_path)
+
+    if os.path.basename(file_path) == "__init__.py":
+        exceptions |= get_noqa_exceptions(file_path)
+
+    if exceptions:
+        new_config = configparser.ConfigParser()
+        new_config.read(flake8_config)  # https://bugs.python.org/issue16058 Why don't use deepcopy
+        new_config["flake8"]["ignore"] += "\n" + "\n".join(x + "," for x in sorted(exceptions))
+
+        config_stream = io.StringIO()
+        new_config.write(config_stream)
+        config_hash = hashlib.md5(config_stream.getvalue().encode()).hexdigest()
+        config_path = config_hash + ".config"
+        if not os.path.exists(config_path):
+            with open(config_path, "w") as f:
+                f.write(config_stream.getvalue())
+        return config_path
+    else:
+        return flake8_config
+
+
+def get_noqa_exceptions(file_path: str) -> set:
+    additional_exceptions = get_file_ignores(file_path)
+    validate_exceptions(additional_exceptions)
+    return additional_exceptions & ALLOWED_IGNORES
+
+
+def get_file_ignores(file_path: str) -> set:
+    file_ignore_regex = re.compile(r"#\s*flake8\s+noqa:\s*(.*)")
+    with open(file_path) as afile:
+        # looking for ignores only in the first 3 lines
+        for line in itertools.islice(afile, 3):
+            if match := file_ignore_regex.search(line):
+                ignores = match.group(1).strip()
+                if ignores:
+                    ignores = re.split(r"\s*,\s*", ignores)
+                    return set(ignores)
+    return set()
+
+
+def validate_exceptions(exceptions: set) -> None:
+    if exceptions - ALLOWED_IGNORES:
+        logger.error(
+            "Disabling %s checks. Only %s can be suppressed in the __init__.py files using # flake8 noqa",
+            str(list(exceptions - ALLOWED_IGNORES)),
+            str(list(ALLOWED_IGNORES)),
+        )
+
+
+def run_flake8_for_dir(flake8_bin: str, source_root: str, config: str, check_files: list[str]) -> dict[str, list[str]]:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        logger.debug("flake8 temp dir: %s", temp_dir)
+        for f in check_files:
+            copy_file_path = os.path.join(temp_dir, os.path.relpath(f, source_root))
+            os.makedirs(os.path.dirname(copy_file_path), exist_ok=True)
+            shutil.copyfile(f, copy_file_path)
+        flake8_res = run_flake8(flake8_bin, temp_dir, config)
+        return get_flake8_results(flake8_res, source_root, temp_dir)
+
+
+def run_flake8(flake8_bin: str, dir_path: str, config: str) -> list[str]:
+    cmd = [flake8_bin, dir_path, config]
+    res = subprocess.run(cmd, capture_output=True, encoding="utf8", errors="replace")
+    if res.stderr:
+        logger.debug("flake8 stderr: %s", res.stderr)
+    return res.stdout.split("\n") if res.returncode else []
+
+
+def get_flake8_results(flake8_res: list[str], source_root: str, temp_dir: str) -> dict[str, list[str]]:
+    flake8_errors_map = defaultdict(list)
+    for line in iterate_over_results(flake8_res):
+        match = re.match(FLAKE_LINE_RE, line)
+        if not match:
+            raise RuntimeError("Cannot parse flake8 output line: '{}'".format(line))
+        file_path, row, col_with_sep, code, text = match.groups()
+        file_path = file_path.replace(temp_dir, source_root)
+        if col_with_sep is None:
+            col_with_sep = ""
+        colorized_line = f"[[unimp]]{file_path}[[rst]]:[[alt2]]{row}[[rst]]:[[alt2]]{col_with_sep}[[rst]] [[[alt1]]{code}[[rst]]] [[bad]]{text}[[rst]]"
+        flake8_errors_map[file_path].append(colorized_line)
+    return flake8_errors_map
+
+
+def iterate_over_results(flake8_res: list[str]) -> Generator[str, None, None]:
+    to_skip = {"[[bad]]", "[[rst]]"}
+    for line in flake8_res:
+        if line and line not in to_skip:
+            yield line
+
+
+def main():
+    params = linter_params.get_params()
+    logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format="%(asctime)s: %(levelname)s: %(message)s")
+
+    flake8_bin = get_flake8_bin(params)
+    flake8_config = params.configs[FLAKE8_CONFIG_INDEX]
+    migrations_config = get_migrations_config(params)
+    source_root = params.source_root
+
+    logger.debug("Constructing flake8 config")
+    config_map = defaultdict(list)
+    report = reporter.LintReport()
+
+    skipped_files = set()
+    for file_path in params.files:
+        config_path = get_flake8_config(flake8_config, migrations_config, source_root, file_path)
+        if config_path:
+            config_map[config_path].append(file_path)
+        else:
+            skipped_files.add(file_path)
+
+    logger.debug("Configuration:\n%s", str(config_map))
+
+    flake8_errors_map = {}
+    for config_path, check_files in config_map.items():
+        flake8_errors_map.update(run_flake8_for_dir(flake8_bin, source_root, config_path, check_files))
+
+    report = reporter.LintReport()
+    for file_path in params.files:
+        if file_path in skipped_files:
+            report.add(file_path, reporter.LintStatus.SKIPPED, "Skipped by config")
+        elif file_path in flake8_errors_map:
+            message = "\n".join(flake8_errors_map[file_path])
+            report.add(file_path, reporter.LintStatus.FAIL, message)
+        else:
+            report.add(file_path, reporter.LintStatus.GOOD)
+    report.dump(params.report_file)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/flake8_linter/bin/tests/stub/__main__.py b/tools/flake8_linter/bin/tests/stub/__main__.py
new file mode 100644
index 00000000000..3bbfcf424b3
--- /dev/null
+++ b/tools/flake8_linter/bin/tests/stub/__main__.py
@@ -0,0 +1,51 @@
+"""
+flake8 emulator. Does the following:
+- read config file (the name is specified in _FLAKE8_STUB_CONFIG env variable)
+- gather launch info and put it into the report file
+- print test data to stdout
+"""
+
+import json
+import os
+import sys
+
+
+STUB_CONFIG_ENV_VAR_NAME = "_FLAKE8_STUB_CONFIG"
+
+
+def main():
+    flake8_bin, test_dir, flake8_config = sys.argv
+    stub_config_file = os.getenv(STUB_CONFIG_ENV_VAR_NAME)
+    with open(stub_config_file) as f:
+        stub_config = json.load(f)
+
+    stub_output = stub_config["output"]
+    launch_report_file = stub_config["report_file"]
+
+    launch_report = get_launch_report(flake8_bin, test_dir, flake8_config)
+    with open(launch_report_file, "a") as f:
+        json.dump(launch_report, f)
+        f.write("\n")
+
+    if stub_output:
+        sys.stdout.write(stub_output.format(test_dir=test_dir))
+        return 1
+    else:
+        return 0
+
+
+def get_launch_report(flake8_bin, test_dir, flake8_config):
+    rel_file_paths = []
+    for root, _, files in os.walk(test_dir):
+        rel_file_paths += [os.path.relpath(os.path.join(root, f), test_dir) for f in files]
+    with open(flake8_config) as f:
+        config_data = f.read()
+    return {
+        "flake8_bin": flake8_bin,
+        "rel_file_paths": rel_file_paths,
+        "config_data": config_data,
+    }
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tools/flake8_linter/bin/tests/stub/ya.make b/tools/flake8_linter/bin/tests/stub/ya.make
new file mode 100644
index 00000000000..8c24917942e
--- /dev/null
+++ b/tools/flake8_linter/bin/tests/stub/ya.make
@@ -0,0 +1,9 @@
+PY3_PROGRAM(flake8)
+
+STYLE_PYTHON()
+
+PY_SRCS(
+    __main__.py
+)
+
+END()
diff --git a/tools/flake8_linter/bin/tests/test_flake8_ver.py b/tools/flake8_linter/bin/tests/test_flake8_ver.py
new file mode 100644
index 00000000000..d2760a8c7d2
--- /dev/null
+++ b/tools/flake8_linter/bin/tests/test_flake8_ver.py
@@ -0,0 +1,30 @@
+import os
+import pytest
+
+from . import util
+from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE
+
+
+@pytest.mark.parametrize(
+    "lint_name, global_resource_var_name",
+    [
+        ("py2_flake8", FLAKE8_PY2_RESOURCE),
+        ("py3_flake8", FLAKE8_PY3_RESOURCE),
+    ],
+)
+def test_flake8_version(lint_name, global_resource_var_name):
+    test_file = "project/test.py"
+    runner = util.LinterRunner(lint_name)
+    runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
+    run_result = runner.run_test([test_file])
+    expected_flake8_bin = os.path.join(runner.flake8_path(global_resource_var_name), "flake8")
+    assert run_result.flake8_launches[0].flake8_bin == expected_flake8_bin
+
+
+def test_raise_on_incorrect_lint_name():
+    test_file = "project/test.py"
+    runner = util.LinterRunner("strange_lint_name")
+    runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
+    run_result = runner.run_test([test_file])
+    assert run_result.linter_run_result.returncode != 0
+    assert "Unexpected lint name" in run_result.linter_run_result.stderr
diff --git a/tools/flake8_linter/bin/tests/test_migrations.py b/tools/flake8_linter/bin/tests/test_migrations.py
new file mode 100644
index 00000000000..1e77d265be5
--- /dev/null
+++ b/tools/flake8_linter/bin/tests/test_migrations.py
@@ -0,0 +1,164 @@
+import pytest
+from configparser import ConfigParser
+
+from . import util
+
+FLAKE8_CONFIG_DATA = """
+    [flake8]
+    select = E, W, F
+    ignore =
+        E122,
+        E743,
+        F403,
+        W605,
+"""
+MIGRATION_CONFIG_DATA = """
+    flake8:
+        F401:
+            ignore:
+            - F401
+            prefixes:
+            - project401
+        SKIP:
+            ignore:
+            - "*"
+            prefixes:
+            - project_skip
+"""
+FILE_WITHOUT_EXCEPTIONS = "project/file1.py"
+FILE_IGNORE_F401 = "project401/file2.py"
+FILE_SKIPPED = "project_skip/file3.py"
+
+
+@pytest.mark.parametrize(
+    "test_file, added_ignore, disable_migrations",
+    [
+        (FILE_WITHOUT_EXCEPTIONS, "", False),
+        (FILE_IGNORE_F401, "\nF401,", False),
+        (FILE_IGNORE_F401, "", True),
+    ],
+)
+def test_ignore(test_file, added_ignore, disable_migrations):
+    test_files = [test_file]
+
+    runner = util.LinterRunner()
+    runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
+    runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA)
+    runner.create_source_tree(test_files)
+
+    disable_migrations_params = {"extra_params": {"DISABLE_FLAKE8_MIGRATIONS": "yes"}} if disable_migrations else {}
+    run_result = runner.run_test(test_files, custom_params=disable_migrations_params)
+
+    assert len(run_result.flake8_launches) == 1
+
+    launch = run_result.flake8_launches[0]
+    got_config = ConfigParser()
+    got_config.read_string(launch.config_data)
+    expected_config = ConfigParser()
+    expected_config.read_string(FLAKE8_CONFIG_DATA)
+    expected_config["flake8"]["ignore"] += added_ignore
+
+    util.assert_configs(got_config, expected_config)
+
+    assert launch.rel_file_paths == test_files
+
+
+def test_skipped():
+    test_files = [FILE_SKIPPED]
+
+    runner = util.LinterRunner()
+    runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
+    runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA)
+    runner.create_source_tree(test_files)
+
+    run_result = runner.run_test(test_files)
+
+    assert len(run_result.flake8_launches) == 0
+
+    abs_test_file_path = runner.abs_source_file_path(FILE_SKIPPED)
+    assert run_result.report_data["report"][abs_test_file_path]["status"] == "SKIPPED"
+
+
+def test_group_files_by_config():
+    test_files = [FILE_WITHOUT_EXCEPTIONS, FILE_IGNORE_F401]
+
+    runner = util.LinterRunner()
+    runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
+    runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, MIGRATION_CONFIG_DATA)
+    runner.create_source_tree(test_files)
+
+    run_result = runner.run_test(test_files)
+
+    assert len(run_result.flake8_launches) == 2
+
+    for launch in run_result.flake8_launches:
+        rel_file_paths = launch.rel_file_paths
+        got_config = ConfigParser()
+        got_config.read_string(launch.config_data)
+        # Relaxed check if config is matched with a checked file
+        # Thorough config check is done in test_ignore()
+        if rel_file_paths == [FILE_WITHOUT_EXCEPTIONS]:
+            assert "F401" not in got_config["flake8"]["ignore"]
+        elif rel_file_paths == [FILE_IGNORE_F401]:
+            assert "F401" in got_config["flake8"]["ignore"]
+        else:
+            pytest.fail("Unexpected file paths passed to flake8 binary: {}".format(rel_file_paths))
+
+
+@pytest.mark.parametrize(
+    "migrations_file, expected_ignore",
+    [
+        (None, "F777"),
+        ("", None),
+        ("build/config/other_migration.yaml", "F888"),
+    ],
+)
+def test_migration_file_from_env(migrations_file, expected_ignore):
+    # Env var _YA_TEST_FLAKE8_CONFIG overrides file name from configs parameter. _YA_TEST_FLAKE8_CONFIG:
+    # - is not defined - use migrations file from configs parameter
+    # - is empty - don't use migrations file at all
+    # - not empty - use variable value as migrations file name
+    config_migrations = """
+        flake8:
+            ignore:
+                ignore:
+                - F777
+                prefixes:
+                - project
+    """
+    test_files = ["project/test.py"]
+
+    runner = util.LinterRunner()
+    runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
+    runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, config_migrations)
+    runner.create_source_tree(test_files)
+    env = {}
+    if migrations_file is not None:
+        if migrations_file:
+            env_var_migrations = """
+                flake8:
+                    ignore:
+                        ignore:
+                        - F888
+                        prefixes:
+                        - project
+            """
+            runner.create_source_file(migrations_file, env_var_migrations)
+            env_var_value = runner.abs_source_file_path(migrations_file)
+        else:
+            env_var_value = ""
+        env["_YA_TEST_FLAKE8_CONFIG"] = env_var_value
+
+    run_result = runner.run_test(test_files, env=env)
+
+    assert len(run_result.flake8_launches) == 1
+
+    launch = run_result.flake8_launches[0]
+    got_config = ConfigParser()
+    got_config.read_string(launch.config_data)
+    ignores = got_config["flake8"]["ignore"]
+    if expected_ignore:
+        assert expected_ignore in ignores
+    else:
+        assert "F777" not in ignores
+        assert "F888" not in ignores
diff --git a/tools/flake8_linter/bin/tests/test_noqa.py b/tools/flake8_linter/bin/tests/test_noqa.py
new file mode 100644
index 00000000000..1f7d536fe72
--- /dev/null
+++ b/tools/flake8_linter/bin/tests/test_noqa.py
@@ -0,0 +1,73 @@
+import io
+import pytest
+from configparser import ConfigParser
+
+from . import util
+
+FLAKE8_CONFIG_DATA = """
+    [flake8]
+    select = E, W, F
+    ignore =
+        E122,
+"""
+
+
+@pytest.mark.parametrize(
+    "file_name, noqa_line_no, is_scanned",
+    [
+        ("__init__.py", 1, True),
+        ("__init__.py", 3, True),
+        ("__init__.py", 4, False),
+        ("not_init.py", 1, False),
+    ],
+)
+def test_scanned_line_count(file_name, noqa_line_no, is_scanned):
+    test_file = "project/" + file_name
+
+    test_file_data = io.StringIO()
+    for lno in range(1, 10):
+        test_file_data.write("pass")
+        if lno == noqa_line_no:
+            test_file_data.write(" # flake8 noqa: F401")
+        test_file_data.write("\n")
+
+    runner = util.LinterRunner()
+    runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
+    runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, "")
+    runner.create_source_file(test_file, test_file_data.getvalue())
+
+    run_result = runner.run_test([test_file])
+
+    assert len(run_result.flake8_launches) == 1
+
+    launch = run_result.flake8_launches[0]
+    got_config = ConfigParser()
+    got_config.read_string(launch.config_data)
+    if is_scanned:
+        assert "F401" in got_config["flake8"]["ignore"]
+    else:
+        assert "F401" not in got_config["flake8"]["ignore"]
+
+
+def test_not_F401():
+    test_file = "project/__init__.py"
+
+    test_file_data = """
+        pass # flake8 noqa: F777, F401
+    """
+
+    runner = util.LinterRunner()
+    runner.create_source_file(util.FLAKE8_CONFIG_FILE, FLAKE8_CONFIG_DATA)
+    runner.create_source_file(util.MIGRATIONS_CONFIG_FILE, "")
+    runner.create_source_file(test_file, test_file_data)
+
+    run_result = runner.run_test([test_file])
+
+    assert len(run_result.flake8_launches) == 1
+
+    launch = run_result.flake8_launches[0]
+    got_config = ConfigParser()
+    got_config.read_string(launch.config_data)
+    assert "F401" in got_config["flake8"]["ignore"]
+    assert "F777" not in got_config["flake8"]["ignore"]
+    assert "Disabling ['F777'] checks" in run_result.linter_run_result.stdout
diff --git a/tools/flake8_linter/bin/tests/test_report.py b/tools/flake8_linter/bin/tests/test_report.py
new file mode 100644
index 00000000000..9304fe86b3b
--- /dev/null
+++ b/tools/flake8_linter/bin/tests/test_report.py
@@ -0,0 +1,101 @@
+import pytest
+
+from . import util
+
+FLAKE8_CONFIG_DATA = """
+    [flake8]
+    select = E, W, F
+    ignore =
+        E122,
+"""
+
+
+def test_no_errors():
+    test_file = "project/test.py"
+    runner = util.LinterRunner()
+    runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
+
+    run_result = runner.run_test([test_file])
+
+    abs_test_file = runner.abs_source_file_path(test_file)
+    file_report = run_result.report_data["report"][abs_test_file]
+    assert file_report["status"] == "GOOD"
+    assert file_report["message"] == ""
+    assert file_report["elapsed"] >= 0.0
+
+
+def test_skip_markup():
+    test_file = "project/test.py"
+    flake8_result = """
+        [[bad]]
+        [[rst]]
+    """
+
+    runner = util.LinterRunner()
+    runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
+
+    run_result = runner.run_test([test_file], flake8_result=flake8_result)
+
+    abs_test_file = runner.abs_source_file_path(test_file)
+    file_report = run_result.report_data["report"][abs_test_file]
+    assert file_report["status"] == "GOOD"
+    assert file_report["message"] == ""
+    assert file_report["elapsed"] >= 0.0
+
+
+@pytest.mark.parametrize(
+    "errors",
+    [
+        [("10", "F401", "Error with row number only")],
+        [("10:20", "F401", "Error with row and column numbers")],
+        [
+            ("10", "F401", "Multiple errors: the first error"),
+            ("20", "F402", "Multiple errors: the second error"),
+        ],
+    ],
+)
+def test_error_formatting(errors):
+    test_file = "project/test.py"
+    flake8_result = "[[bad]]\n"
+    for file_pos, code, text in errors:
+        flake8_result += f"{{test_dir}}/{test_file}:{file_pos}: [{code}] {text}\n"
+    flake8_result += "[[rst]]\n"
+
+    runner = util.LinterRunner()
+    runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
+
+    run_result = runner.run_test([test_file], flake8_result=flake8_result)
+
+    abs_test_file = runner.abs_source_file_path(test_file)
+    file_report = run_result.report_data["report"][abs_test_file]
+    expected_message_lines = []
+    for file_pos, code, text in errors:
+        if ":" in file_pos:
+            row, col = file_pos.split(":")
+            col_with_sep = col + ":"
+        else:
+            row = file_pos
+            col_with_sep = ""
+        line = f"[[unimp]]{abs_test_file}[[rst]]:[[alt2]]{row}[[rst]]:[[alt2]]{col_with_sep}[[rst]] [[[alt1]]{code}[[rst]]] [[bad]]{text}[[rst]]"
+        expected_message_lines.append(line)
+
+    assert file_report["status"] == "FAIL"
+    assert file_report["message"] == "\n".join(expected_message_lines)
+    assert file_report["elapsed"] >= 0.0
+
+
+def test_fail_on_wrong_message():
+    test_file = "project/test.py"
+    flake8_result = """
+        [[bad]]
+        Unexpected error message
+        [[rst]]
+    """
+
+    runner = util.LinterRunner()
+    runner.create_source_tree(util.DEFAULT_CONFIGS + [test_file])
+
+    run_result = runner.run_test([test_file], flake8_result=flake8_result)
+
+    assert run_result.linter_run_result.returncode != 0
+    assert "Cannot parse flake8 output line" in run_result.linter_run_result.stderr
diff --git a/tools/flake8_linter/bin/tests/util.py b/tools/flake8_linter/bin/tests/util.py
new file mode 100644
index 00000000000..be3912e2bce
--- /dev/null
+++ b/tools/flake8_linter/bin/tests/util.py
@@ -0,0 +1,151 @@
+import json
+import logging
+import mergedeep
+import os
+import shutil
+import subprocess
+import tempfile
+from configparser import ConfigParser
+from dataclasses import dataclass
+from textwrap import dedent
+
+from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE
+from yatest.common import work_path, binary_path
+
+# Config paths to reuse in different tests (just for convenience). This is not mandatory config paths.
+FLAKE8_CONFIG_FILE = "build/config/flake8.cfg"
+MIGRATIONS_CONFIG_FILE = "build/config/migrations.yaml"
+DEFAULT_CONFIGS = [FLAKE8_CONFIG_FILE, MIGRATIONS_CONFIG_FILE]
+
+# Pass test parameters to flake8 stub via env variable
+STUB_CONFIG_ENV_VAR_NAME = "_FLAKE8_STUB_CONFIG"
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Flake8Launch:
+    flake8_bin: str
+    rel_file_paths: list[str]
+    config_data: str
+
+
+@dataclass
+class RunTestResult:
+    flake8_launches: list[Flake8Launch]
+    linter_run_result: subprocess.CompletedProcess
+    report_data: dict
+
+
+class LinterRunner:
+    def __init__(self, lint_name: str = "py3_flake8"):
+        self._lint_name = lint_name
+        self._source_root = tempfile.mkdtemp(prefix="source_root", dir=work_path())
+        self._work_root = tempfile.mkdtemp(prefix="work_root", dir=work_path())
+        self._params_file = os.path.join(self._work_root, "params.json")
+        self._report_file = os.path.join(self._work_root, "report.json")
+        self._launch_report_file = os.path.join(self._work_root, "launches.json")
+        self._stub_config_file = os.path.join(self._work_root, "stub_config.json")
+        self._linter_path = binary_path("tools/flake8_linter/bin/flake8_linter")
+        self._global_resources = self._prepare_global_resources()
+
+    def _prepare_global_resources(self):
+        global_resource_root = tempfile.mkdtemp(prefix="global_resources", dir=work_path())
+        py2_stub_path = os.path.join(global_resource_root, "py2")
+        py3_stub_path = os.path.join(global_resource_root, "py3")
+        stub_path = binary_path("tools/flake8_linter/bin/tests/stub")
+        shutil.copytree(stub_path, py2_stub_path, copy_function=os.link)
+        shutil.copytree(stub_path, py3_stub_path, copy_function=os.link)
+        return {
+            FLAKE8_PY2_RESOURCE: py2_stub_path,
+            FLAKE8_PY3_RESOURCE: py3_stub_path,
+        }
+
+    def create_source_file(self, rel_file_path: str, data: str):
+        abs_file_path = os.path.join(self._source_root, rel_file_path)
+        os.makedirs(os.path.dirname(abs_file_path), exist_ok=True)
+        with open(abs_file_path, "w") as f:
+            f.write(data)
+
+    def create_source_tree(self, rel_file_paths: list[str]):
+        for rel_file_path in rel_file_paths:
+            self.create_source_file(rel_file_path, "")
+
+    def abs_source_file_path(self, rel_file_path: str):
+        return os.path.join(self._source_root, rel_file_path)
+
+    def run_test(
+        self,
+        file_rel_paths: list[str],
+        config_rel_paths: list[str] = DEFAULT_CONFIGS,
+        flake8_result: str = "",
+        custom_params: dict = {},
+        env: dict[str, str] = {},
+    ) -> RunTestResult:
+        self._prepare_params(config_rel_paths, file_rel_paths, custom_params)
+        stub_config = {
+            "output": dedent(flake8_result),
+            "report_file": self._launch_report_file,
+        }
+        stub_env = {
+            STUB_CONFIG_ENV_VAR_NAME: self._stub_config_file,
+        }
+        run_env = mergedeep.merge({}, env, stub_env)
+        with open(self._stub_config_file, "w") as f:
+            json.dump(stub_config, f)
+        linter_run_result = subprocess.run(
+            [self._linter_path, "--params", self._params_file],
+            encoding="utf-8",
+            capture_output=True,
+            check=False,
+            env=run_env,
+        )
+        logger.debug("Linter run result: %s", str(linter_run_result))
+
+        if os.path.exists(self._report_file):
+            with open(self._report_file) as f:
+                report_data = json.load(f)
+        else:
+            report_data = None
+
+        return RunTestResult(self._read_launches(), linter_run_result, report_data)
+
+    def flake8_path(self, global_resource_var_name):
+        return self._global_resources[global_resource_var_name]
+
+    def _prepare_params(self, config_rel_paths: list[str], file_rel_paths: list[str], custom_params: dict):
+        params = {
+            "source_root": self._source_root,
+            "project_path": "",
+            "output_path": "",
+            "lint_name": self._lint_name,
+            "depends": {},
+            "global_resources": self._global_resources,
+            "configs": self._mk_source_abs_path(config_rel_paths),
+            "report_file": self._report_file,
+            "files": self._mk_source_abs_path(file_rel_paths),
+        }
+        mergedeep.merge(params, custom_params)
+        with open(self._params_file, "w") as f:
+            json.dump(params, f)
+
+    def _mk_source_abs_path(self, paths):
+        return [self.abs_source_file_path(p) for p in paths]
+
+    def _read_launches(self):
+        launches = []
+        if os.path.exists(self._launch_report_file):
+            with open(self._launch_report_file) as f:
+                for line in f:
+                    logger.debug("Launch report line: %s", line)
+                    launch = json.loads(line)
+                    launches.append(Flake8Launch(**launch))
+        else:
+            logger.debug("Launch report file not found: %s", self._launch_report_file)
+        return launches
+
+
+def assert_configs(got: ConfigParser, expected: ConfigParser):
+    got_dict = dict(got["flake8"].items())
+    expected_dict = dict(expected["flake8"].items())
+    assert got_dict == expected_dict
diff --git a/tools/flake8_linter/bin/tests/ya.make b/tools/flake8_linter/bin/tests/ya.make
new file mode 100644
index 00000000000..533a02a1f12
--- /dev/null
+++ b/tools/flake8_linter/bin/tests/ya.make
@@ -0,0 +1,29 @@
+PY3TEST()
+
+STYLE_PYTHON()
+
+TEST_SRCS(
+    test_flake8_ver.py
+    test_migrations.py
+    test_noqa.py
+    test_report.py
+    util.py
+)
+
+PEERDIR(
+    build/plugins/lib
+    contrib/python/mergedeep
+    devtools/ya/test/tests/lib/common
+    library/python/testing/custom_linter_util
+)
+
+DEPENDS(
+    tools/flake8_linter/bin
+    tools/flake8_linter/bin/tests/stub
+)
+
+END()
+
+RECURSE(
+    stub
+)
diff --git a/tools/flake8_linter/bin/ya.make b/tools/flake8_linter/bin/ya.make
new file mode 100644
index 00000000000..197c4b94044
--- /dev/null
+++ b/tools/flake8_linter/bin/ya.make
@@ -0,0 +1,20 @@
+PY3_PROGRAM(flake8_linter)
+
+STYLE_PYTHON()
+
+PEERDIR(
+    build/plugins/lib/test_const
+    devtools/ya/test/programs/test_tool/lib/migrations_config
+    devtools/ya/yalibrary/term
+    library/python/testing/custom_linter_util
+)
+
+SRCDIR(
+    tools/flake8_linter/bin
+)
+
+PY_SRCS(
+    __main__.py
+)
+
+END()
diff --git a/tools/flake8_linter/ya.make b/tools/flake8_linter/ya.make
new file mode 100644
index 00000000000..eac0ce5f218
--- /dev/null
+++ b/tools/flake8_linter/ya.make
@@ -0,0 +1,15 @@
+IF (USE_PREBUILT_TOOLS OR OPENSOURCE)
+    INCLUDE(${ARCADIA_ROOT}/build/prebuilt/tools/flake8_linter/ya.make.prebuilt)
+ENDIF()
+
+IF (NOT PREBUILT)
+    INCLUDE(${ARCADIA_ROOT}/tools/flake8_linter/bin/ya.make)
+ENDIF()
+
+RECURSE(
+    bin
+)
+
+RECURSE_FOR_TESTS(
+    bin/tests
+)