diff options
Diffstat (limited to 'tools')
24 files changed, 758 insertions, 85 deletions
diff --git a/tools/black_linter/wrapper.py b/tools/black_linter/wrapper.py new file mode 100644 index 00000000000..6f11136d356 --- /dev/null +++ b/tools/black_linter/wrapper.py @@ -0,0 +1,104 @@ +import logging +import os +import subprocess +import sys +import time +from pathlib import Path + +from build.plugins.lib.test_const import BLACK_RESOURCE +from library.python.testing.custom_linter_util import linter_params, reporter +from library.python.testing.style import rules + +logger = logging.getLogger(__name__) + +SNIPPET_LINES_LIMIT = 20 + + +def get_black_bin(params) -> str: + black_root = params.global_resources[BLACK_RESOURCE] + return os.path.join(black_root, 'black') + + +def run_black(black_bin, filename, args): + cmd = [black_bin, filename, *args] + + res = subprocess.run( + cmd, + capture_output=True, + encoding='utf8', + errors='replace', + ) + + return res.returncode, res.stdout if res.returncode else '' + + +def run_black_safe(black_bin, filename, args): + try: + return run_black(black_bin, filename, args) + except Exception: + # fast mode failed - retry + return run_black(black_bin, filename, args + ['--fast']) + + +def process_file(black_bin, filename, config): + logger.debug("Check %s", filename) + args = ['--quiet', '--check', '--config', config] + + # Fast path for runs with fix_style option or without errors. + rc, out = run_black_safe(black_bin, filename, args) + if rc == 1: + # black runs 15x+ slower if diff is requested, even for files w/o actual diff. + # Rerun black in case of found error. + rc, out = run_black_safe(black_bin, filename, args + ['--diff']) + + if out: + sys.stdout.write(out) + lines = out.splitlines(keepends=True) + # strip diff header with "+++" "---" lines + lines = lines[2:] + if len(lines) > SNIPPET_LINES_LIMIT: + lines = lines[:SNIPPET_LINES_LIMIT] + lines += ["[[rst]]..[truncated].. see full diff in the stdout file in the logsdir"] + out = ''.join(lines) + + return out + + +def main(): + params = linter_params.get_params() + + black_bin = get_black_bin(params) + style_config_path = Path(params.source_root, params.configs[0]) + + report = reporter.LintReport() + for file_name in params.files: + start_time = time.perf_counter() + + skip_reason = rules.get_skip_reason(file_name, Path(file_name).read_text(), skip_links=False) + if skip_reason: + elapsed = time.perf_counter() - start_time + report.add( + file_name, + reporter.LintStatus.SKIPPED, + f"Style check is omitted: {skip_reason}", + elapsed=elapsed, + ) + continue + + error = process_file(black_bin, file_name, style_config_path) + elapsed = time.perf_counter() - start_time + + if error: + rel_file_name = os.path.relpath(file_name, params.source_root) + message = "Run [[imp]]ya style {}[[rst]] to fix format\n".format(rel_file_name) + error + status = reporter.LintStatus.FAIL + else: + message = "" + status = reporter.LintStatus.GOOD + report.add(file_name, status, message, elapsed=elapsed) + + report.dump(params.report_file) + + +if __name__ == "__main__": + main() diff --git a/tools/black_linter/ya.make b/tools/black_linter/ya.make deleted file mode 100644 index ed2d6421328..00000000000 --- a/tools/black_linter/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -IF (USE_PREBUILT_TOOLS OR OPENSOURCE) - INCLUDE(${ARCADIA_ROOT}/build/prebuilt/tools/black_linter/ya.make.prebuilt) -ENDIF() - -IF (NOT PREBUILT) - INCLUDE(${ARCADIA_ROOT}/tools/black_linter/bin/ya.make) -ENDIF() - -IF (NOT OPENSOURCE) - RECURSE( - bin - ) -ENDIF() diff --git a/tools/cpp_style_checker/__main__.py b/tools/cpp_style_checker/wrapper.py index 4ca1bc3a0f9..fe09bc6cb3c 100644 --- a/tools/cpp_style_checker/__main__.py +++ b/tools/cpp_style_checker/wrapper.py @@ -1,9 +1,7 @@ import difflib -import json import os import subprocess import time -import yaml from pathlib import PurePath from build.plugins.lib.test_const import CLANG_FORMAT_RESOURCE @@ -37,21 +35,17 @@ def main(): style_config_path = params.configs[0] - with open(style_config_path) as f: - style_config = yaml.safe_load(f) - style_config_json = json.dumps(style_config) - report = reporter.LintReport() for file_name in params.files: - start_time = time.time() - status, message = check_file(clang_format_binary, style_config_json, file_name) - elapsed = time.time() - start_time + start_time = time.perf_counter() + status, message = check_file(clang_format_binary, style_config_path, file_name) + elapsed = time.perf_counter() - start_time report.add(file_name, status, message, elapsed=elapsed) report.dump(params.report_file) -def check_file(clang_format_binary, style_config_json, filename): +def check_file(clang_format_binary, style_config_path, filename): with open(filename, "rb") as f: actual_source = f.read() @@ -59,7 +53,7 @@ def check_file(clang_format_binary, style_config_json, filename): if skip_reason: return reporter.LintStatus.SKIPPED, "Style check is omitted: {}".format(skip_reason) - command = [clang_format_binary, '-assume-filename=' + filename, '-style=' + style_config_json] + command = [clang_format_binary, '-assume-filename=' + filename, '-style=file:' + style_config_path] styled_source = subprocess.check_output(command, input=actual_source) if styled_source == actual_source: diff --git a/tools/cpp_style_checker/ya.make b/tools/cpp_style_checker/ya.make deleted file mode 100644 index e762be1a1fc..00000000000 --- a/tools/cpp_style_checker/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -PY3_PROGRAM() - -PEERDIR( - build/plugins/lib/test_const - contrib/python/PyYAML - library/python/testing/custom_linter_util - library/python/testing/style -) - -PY_SRCS( - __main__.py -) - -STYLE_PYTHON() - -END() diff --git a/tools/enum_parser/enum_parser/main.cpp b/tools/enum_parser/enum_parser/main.cpp index 433a61a5db9..c87f76e98dc 100644 --- a/tools/enum_parser/enum_parser/main.cpp +++ b/tools/enum_parser/enum_parser/main.cpp @@ -269,8 +269,8 @@ void GenerateEnum( out << " NAMES_INITIALIZATION_PAIRS,\n"; out << " VALUES_INITIALIZATION_PAIRS,\n"; out << " CPP_NAMES_INITIALIZATION_ARRAY,\n"; - out << " " << WrapStringBuf(outerScopeStr) << ",\n"; - out << " " << WrapStringBuf(name) << "\n"; + out << " ::NEnumSerializationRuntime::CommonStringBuffer(" << WrapStringBuf(outerScopeStr) << ", " << WrapStringBuf(name) << ").first,\n"; + out << " ::NEnumSerializationRuntime::CommonStringBuffer(" << WrapStringBuf(outerScopeStr) << ", " << WrapStringBuf(name) << ").second,\n"; out << " };\n\n"; // Properties diff --git a/tools/enum_parser/enum_parser/ya.make b/tools/enum_parser/enum_parser/ya.make index 1bd2356ff15..a0608e4e30a 100644 --- a/tools/enum_parser/enum_parser/ya.make +++ b/tools/enum_parser/enum_parser/ya.make @@ -1,5 +1,8 @@ PROGRAM(enum_parser) +NO_PROFILE_RUNTIME() +NO_CLANG_COVERAGE() + SRCS( main.cpp ) diff --git a/tools/enum_parser/enum_serialization_runtime/README.md b/tools/enum_parser/enum_serialization_runtime/README.md index b8b4dd92cf6..05c30f3ec41 100644 --- a/tools/enum_parser/enum_serialization_runtime/README.md +++ b/tools/enum_parser/enum_serialization_runtime/README.md @@ -22,5 +22,5 @@ Use `GENERATE_ENUM_SERIALIZATION_WITH_HEADER` and `GENERATE_ENUM_SERIALIZATION` 3) как следствие, он меньше засоряет во время исполнения кеш инструкций процессора одинаковыми или очень похожими специализациями функций. -Преобразование между `enum` и `int` выносится в пользовательский код (в шаблонные `inline` функции), и производится только в момент непосредственного использования (первым действием в семействе функций `ToString`, последним действием в семуйстве функций `FromString`), где оптимизирующий компилятор обычно может заменить их на no-op или на простые операции со значениями в регистрах. +Преобразование между `enum` и `int` выносится в пользовательский код (в шаблонные `inline` функции), и производится только в момент непосредственного использования (первым действием в семействе функций `ToString`, последним действием в семействе функций `FromString`), где оптимизирующий компилятор обычно может заменить их на no-op или на простые операции со значениями в регистрах. А контейнеры вида `TVector<EEnum>` и `TMap<EEnum, ...>`, которые возвращаются из функций `util/generic/serialized_enum.h`, заменяются на специальные классы `TArrayView` и `TMappedDictView`. Они также поддерживают быстрое и преобразование из перечислений в целочисленные типы и обратно в момент использования, и не требуют создавать специализации для каждого из возможных типов-перечислений. diff --git a/tools/enum_parser/enum_serialization_runtime/enum_runtime.h b/tools/enum_parser/enum_serialization_runtime/enum_runtime.h index 7c85ed3934a..7362c3d243b 100644 --- a/tools/enum_parser/enum_serialization_runtime/enum_runtime.h +++ b/tools/enum_parser/enum_serialization_runtime/enum_runtime.h @@ -173,4 +173,16 @@ namespace NEnumSerializationRuntime { return {TCast::CastToRepresentationType(key), name}; } }; + + constexpr std::pair<TStringBuf, TStringBuf> CommonStringBuffer(const TStringBuf a, const TStringBuf b) { + if (!a.empty() && !b.empty()) { + if (a.StartsWith(b)) { + return {a, TStringBuf{a.data(), b.size()}}; + } + if (b.StartsWith(a)) { + return {TStringBuf{b.data(), a.size()}, b}; + } + } + return {a, b}; + } } diff --git a/tools/enum_parser/parse_enum/parse_enum_ut.cpp b/tools/enum_parser/parse_enum/parse_enum_ut.cpp index e979f5119a8..5f7972d9a74 100644 --- a/tools/enum_parser/parse_enum/parse_enum_ut.cpp +++ b/tools/enum_parser/parse_enum/parse_enum_ut.cpp @@ -343,7 +343,8 @@ Y_UNIT_TEST_SUITE(TEnumParserTest) { const TEnum& e = enums[0]; UNIT_ASSERT_VALUES_EQUAL(e.CppName, "ELiterals"); static constexpr TNameValuePair ref[]{ - {"Char", "sizeof(u8'.')"}, + {"Char1", "sizeof(u8'.')"}, + {"Char2", "sizeof(u8'0')"}, {"Int", "123'456'789"}, {"Float1", "int(456'789.123'456)"}, {"Float2", "int(1'2e0'1)"}, diff --git a/tools/enum_parser/parse_enum/ut/digit_separator.h b/tools/enum_parser/parse_enum/ut/digit_separator.h index c7c8f526acc..d9d4a0ef38a 100644 --- a/tools/enum_parser/parse_enum/ut/digit_separator.h +++ b/tools/enum_parser/parse_enum/ut/digit_separator.h @@ -1,7 +1,8 @@ #pragma once enum class ELiterals { - Char = sizeof(u8'.'), + Char1 = sizeof(u8'.'), + Char2 = sizeof(u8'0'), Int = 123'456'789, Float1 = int(456'789.123'456), Float2 = int(1'2e0'1), diff --git a/tools/flake8_linter/__main__.py b/tools/flake8_linter/__main__.py new file mode 100644 index 00000000000..26e94564b1d --- /dev/null +++ b/tools/flake8_linter/__main__.py @@ -0,0 +1,193 @@ +import configparser +import hashlib +import itertools +import io +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +from collections import defaultdict +from typing import Generator + +from devtools.ya.test.programs.test_tool.lib.migrations_config import load_yaml_config, MigrationsConfig +from library.python.testing.custom_linter_util import linter_params, reporter +from build.plugins.lib.test_const import FLAKE8_PY2_RESOURCE, FLAKE8_PY3_RESOURCE + +logger = logging.getLogger(__name__) + +ALLOWED_IGNORES = {"F401"} +# Supports both default and pylint formats +FLAKE_LINE_RE = r"^(.*?):(\d+):(\d+:)? \[(\w+\d+)\] (.*)" +FLAKE8_CONFIG_INDEX = 0 +MIGRATIONS_CONFIG_INDEX = 1 + + +def get_flake8_bin(params) -> str: + if params.lint_name == "py2_flake8": + flake8_root = params.global_resources[FLAKE8_PY2_RESOURCE] + elif params.lint_name == "flake8": + flake8_root = params.global_resources[FLAKE8_PY3_RESOURCE] + else: + raise RuntimeError("Unexpected lint name: {}".format(params.lint_name)) + return os.path.join(flake8_root, "flake8") + + +def get_migrations_config(params) -> MigrationsConfig: + if params.extra_params.get("DISABLE_FLAKE8_MIGRATIONS", "no") == "yes": + return MigrationsConfig() + config_path = os.getenv("_YA_TEST_FLAKE8_CONFIG") + if config_path is None and len(params.configs) > 1: + config_path = params.configs[MIGRATIONS_CONFIG_INDEX] + + if not config_path: + return MigrationsConfig() + else: + logger.debug("Loading flake8 migrations: %s", config_path) + migrations = load_yaml_config(config_path) + logger.debug("Building migration config") + return MigrationsConfig(migrations) + + +def get_flake8_config( + flake8_config: str, migrations_config: MigrationsConfig, source_root: str, file_path: str +) -> str | None: + arc_rel_file_path = os.path.relpath(file_path, source_root) + if migrations_config.is_skipped(arc_rel_file_path): + return None + exceptions = migrations_config.get_exceptions(arc_rel_file_path) + if exceptions: + logger.info("Ignore flake8 exceptions %s for file %s", str(list(exceptions)), arc_rel_file_path) + + if os.path.basename(file_path) == "__init__.py": + exceptions |= get_noqa_exceptions(file_path) + + if exceptions: + new_config = configparser.ConfigParser() + new_config.read(flake8_config) # https://bugs.python.org/issue16058 Why don't use deepcopy + new_config["flake8"]["ignore"] += "\n" + "\n".join(x + "," for x in sorted(exceptions)) + + config_stream = io.StringIO() + new_config.write(config_stream) + config_hash = hashlib.md5(config_stream.getvalue().encode()).hexdigest() + config_path = config_hash + ".config" + if not os.path.exists(config_path): + with open(config_path, "w") as f: + f.write(config_stream.getvalue()) + return config_path + else: + return flake8_config + + +def get_noqa_exceptions(file_path: str) -> set: + additional_exceptions = get_file_ignores(file_path) + validate_exceptions(additional_exceptions) + return additional_exceptions & ALLOWED_IGNORES + + +def get_file_ignores(file_path: str) -> set: + file_ignore_regex = re.compile(r"#\s*flake8\s+noqa:\s*(.*)") + with open(file_path) as afile: + # looking for ignores only in the first 3 lines + for line in itertools.islice(afile, 3): + if match := file_ignore_regex.search(line): + ignores = match.group(1).strip() + if ignores: + ignores = re.split(r"\s*,\s*", ignores) + return set(ignores) + return set() + + +def validate_exceptions(exceptions: set) -> None: + if exceptions - ALLOWED_IGNORES: + logger.error( + "Disabling %s checks. Only %s can be suppressed in the __init__.py files using # flake8 noqa", + str(list(exceptions - ALLOWED_IGNORES)), + str(list(ALLOWED_IGNORES)), + ) + + +def run_flake8_for_dir(flake8_bin: str, source_root: str, config: str, check_files: list[str]) -> dict[str, list[str]]: + with tempfile.TemporaryDirectory() as temp_dir: + logger.debug("flake8 temp dir: %s", temp_dir) + for f in check_files: + copy_file_path = os.path.join(temp_dir, os.path.relpath(f, source_root)) + os.makedirs(os.path.dirname(copy_file_path), exist_ok=True) + shutil.copyfile(f, copy_file_path) + flake8_res = run_flake8(flake8_bin, temp_dir, config) + return get_flake8_results(flake8_res, source_root, temp_dir) + + +def run_flake8(flake8_bin: str, dir_path: str, config: str) -> list[str]: + cmd = [flake8_bin, dir_path, config] + res = subprocess.run(cmd, capture_output=True, encoding="utf8", errors="replace") + if res.stderr: + logger.debug("flake8 stderr: %s", res.stderr) + return res.stdout.split("\n") if res.returncode else [] + + +def get_flake8_results(flake8_res: list[str], source_root: str, temp_dir: str) -> dict[str, list[str]]: + flake8_errors_map = defaultdict(list) + for line in iterate_over_results(flake8_res): + match = re.match(FLAKE_LINE_RE, line) + if not match: + raise RuntimeError("Cannot parse flake8 output line: '{}'".format(line)) + file_path, row, col_with_sep, code, text = match.groups() + file_path = file_path.replace(temp_dir, source_root) + if col_with_sep is None: + col_with_sep = "" + colorized_line = f"[[unimp]]{file_path}[[rst]]:[[alt2]]{row}[[rst]]:[[alt2]]{col_with_sep}[[rst]] [[[alt1]]{code}[[rst]]] [[bad]]{text}[[rst]]" + flake8_errors_map[file_path].append(colorized_line) + return flake8_errors_map + + +def iterate_over_results(flake8_res: list[str]) -> Generator[str, None, None]: + to_skip = {"[[bad]]", "[[rst]]"} + for line in flake8_res: + if line and line not in to_skip: + yield line + + +def main(): + params = linter_params.get_params() + logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format="%(asctime)s: %(levelname)s: %(message)s") + + flake8_bin = get_flake8_bin(params) + flake8_config = params.configs[FLAKE8_CONFIG_INDEX] + migrations_config = get_migrations_config(params) + source_root = params.source_root + + logger.debug("Constructing flake8 config") + config_map = defaultdict(list) + report = reporter.LintReport() + + skipped_files = set() + for file_path in params.files: + config_path = get_flake8_config(flake8_config, migrations_config, source_root, file_path) + if config_path: + config_map[config_path].append(file_path) + else: + skipped_files.add(file_path) + + logger.debug("Configuration:\n%s", str(config_map)) + + flake8_errors_map = {} + for config_path, check_files in config_map.items(): + flake8_errors_map.update(run_flake8_for_dir(flake8_bin, source_root, config_path, check_files)) + + report = reporter.LintReport() + for file_path in params.files: + if file_path in skipped_files: + report.add(file_path, reporter.LintStatus.SKIPPED, "Skipped by config") + elif file_path in flake8_errors_map: + message = "\n".join(flake8_errors_map[file_path]) + report.add(file_path, reporter.LintStatus.FAIL, message) + else: + report.add(file_path, reporter.LintStatus.GOOD) + report.dump(params.report_file) + + +if __name__ == "__main__": + main() diff --git a/tools/flake8_linter/ya.make b/tools/flake8_linter/ya.make deleted file mode 100644 index cd0469d8b62..00000000000 --- a/tools/flake8_linter/ya.make +++ /dev/null @@ -1,17 +0,0 @@ -IF (USE_PREBUILT_TOOLS OR OPENSOURCE) - INCLUDE(${ARCADIA_ROOT}/build/prebuilt/tools/flake8_linter/ya.make.prebuilt) -ENDIF() - -IF (NOT PREBUILT) - INCLUDE(${ARCADIA_ROOT}/tools/flake8_linter/bin/ya.make) -ENDIF() - -IF (NOT OPENSOURCE) - RECURSE( - bin - ) - - RECURSE_FOR_TESTS( - bin/tests - ) -ENDIF() diff --git a/tools/py2cc/__main__.py b/tools/py2cc/__main__.py new file mode 100755 index 00000000000..54f87baa5ea --- /dev/null +++ b/tools/py2cc/__main__.py @@ -0,0 +1,19 @@ +import marshal +import sys + + +def main(): + srcpathx, in_fname, out_fname = sys.argv[1:] + srcpath = srcpathx[:-1] + + with open(in_fname, 'r') as in_file: + source = in_file.read() + + code = compile(source, srcpath, 'exec', dont_inherit=True) + + with open(out_fname, 'wb') as out_file: + marshal.dump(code, out_file) + + +if __name__ == '__main__': + main() diff --git a/tools/py2cc/bin/ya.make b/tools/py2cc/bin/ya.make new file mode 100644 index 00000000000..bbf86739459 --- /dev/null +++ b/tools/py2cc/bin/ya.make @@ -0,0 +1,29 @@ +PY2_PROGRAM(py2cc) + +#ENABLE(PYBUILD_NO_PYC) + +ENABLE(USE_LIGHT_PY2CC) +DISABLE(PYTHON_SQLITE3) + +PEERDIR( + library/python/runtime + library/python/runtime/main +) + +NO_CHECK_IMPORTS() + +NO_PYTHON_INCLUDES() + +NO_PYTHON_COVERAGE() + +NO_IMPORT_TRACING() + +SRCDIR( + tools/py2cc +) + +PY_SRCS( + __main__.py +) + +END() diff --git a/tools/py2cc/stage0pycc/main.cpp b/tools/py2cc/stage0pycc/main.cpp new file mode 100644 index 00000000000..1c296267f99 --- /dev/null +++ b/tools/py2cc/stage0pycc/main.cpp @@ -0,0 +1,60 @@ +#include <util/folder/path.h> +#include <util/generic/scope.h> +#include <util/generic/string.h> +#include <util/stream/file.h> +#include <util/stream/output.h> + +#include <Python.h> +#include <marshal.h> + +#include <cstdio> +#include <system_error> + +struct TPyObjDeleter { + static void Destroy(PyObject* o) noexcept { + Py_XDECREF(o); + } +}; +using TPyObject = THolder<PyObject, TPyObjDeleter>; + +int main(int argc, char** argv) { + if (argc < 4) { + Cerr << "Usage: " << argv[0] << " SRC_PATH_X SRC OUT" << Endl; + return 1; + } + + TString srcpath{argv[1]}; + srcpath.pop_back(); + const TFsPath inPath{argv[2]}; + const char* outPath = argv[3]; + + Py_Initialize(); + Y_SCOPE_EXIT() { + Py_Finalize(); + }; + + TPyObject bytecode{Py_CompileString( + TFileInput{inPath}.ReadAll().c_str(), + srcpath.c_str(), + Py_file_input)}; + if (!bytecode) { + Cerr << "Failed to compile " << outPath << Endl; + PyErr_Print(); + return 1; + } + + if (FILE* out = fopen(outPath, "wb")) { + PyMarshal_WriteObjectToFile(bytecode.Get(), out, Py_MARSHAL_VERSION); + fclose(out); + if (PyErr_Occurred()) { + Cerr << "Failed to marshal " << outPath << Endl; + PyErr_Print(); + return 1; + } + } else { + Cerr << "Failed to write " << outPath << ": " << std::error_code{errno, std::system_category()}.message() << Endl; + return 1; + } + + return 0; +} diff --git a/tools/py2cc/stage0pycc/ya.make b/tools/py2cc/stage0pycc/ya.make new file mode 100644 index 00000000000..4ac109f0f2b --- /dev/null +++ b/tools/py2cc/stage0pycc/ya.make @@ -0,0 +1,11 @@ +PROGRAM(py2cc) + +PYTHON2_ADDINCL() + +PEERDIR( + contrib/tools/python/lib +) + +SRCS(main.cpp) + +END() diff --git a/tools/py2cc/ya.make b/tools/py2cc/ya.make new file mode 100644 index 00000000000..b6f5ce202eb --- /dev/null +++ b/tools/py2cc/ya.make @@ -0,0 +1,11 @@ +IF (USE_PREBUILT_TOOLS) + INCLUDE(ya.make.prebuilt) +ENDIF() + +IF (NOT PREBUILT) + INCLUDE(bin/ya.make) +ENDIF() + +RECURSE( + bin +) diff --git a/tools/py3cc/bin/ya.make b/tools/py3cc/bin/ya.make index 40e76e93705..555cee8b99e 100644 --- a/tools/py3cc/bin/ya.make +++ b/tools/py3cc/bin/ya.make @@ -1,28 +1,29 @@ -PY3_PROGRAM_BIN(py3cc) +PROGRAM(py3cc) -ENABLE(PYBUILD_NO_PYC) +IF (NOT USE_ARCADIA_PYTHON) + PYTHON3_ADDINCL() -DISABLE(PYTHON_SQLITE3) + PEERDIR( + contrib/tools/python3 + ) +ELSEIF (USE_PYTHON3_PREV) + PEERDIR( + contrib/tools/python3_prev + ) + ADDINCL( + contrib/tools/python3_prev/Include + ) +ELSE() + PEERDIR( + contrib/tools/python3 + ) + ADDINCL( + contrib/tools/python3/Include + ) +ENDIF() -PEERDIR( - library/python/runtime_py3 - library/python/runtime_py3/main -) +SRCDIR(tools/py3cc) -NO_CHECK_IMPORTS() - -NO_PYTHON_INCLUDES() - -NO_PYTHON_COVERAGE() - -NO_IMPORT_TRACING() - -SRCDIR( - tools/py3cc -) - -PY_SRCS( - MAIN main.py -) +SRCS(main.cpp) END() diff --git a/tools/py3cc/main.cpp b/tools/py3cc/main.cpp new file mode 100644 index 00000000000..a70a5ac48ec --- /dev/null +++ b/tools/py3cc/main.cpp @@ -0,0 +1,83 @@ +#include <util/folder/path.h> +#include <util/generic/scope.h> +#include <util/generic/string.h> +#include <util/stream/file.h> +#include <util/stream/output.h> +#include <util/system/shellcommand.h> + +#include <Python.h> +#include <marshal.h> + +#include <cstdio> +#include <system_error> + +struct TPyObjDeleter { + static void Destroy(PyObject* o) noexcept { + Py_XDECREF(o); + } +}; +using TPyObject = THolder<PyObject, TPyObjDeleter>; + +int runSlowPy3cc(const char* slowPy3cc, const char* srcpath, const char* inPath, const char* outPath) { + TShellCommandOptions opts; + opts.SetUseShell(false); + opts.SetOutputStream(&Cout); + opts.SetErrorStream(&Cerr); + + TShellCommand cmd(slowPy3cc, {srcpath, inPath, outPath}, opts); + cmd.Run().Wait(); + + if (auto rc = cmd.GetExitCode(); rc.Defined()) { + return *rc; + } + return 1; +} + +int main(int argc, char** argv) { + if (argc != 6) { + Cerr << "Usage:\n\t" << argv[0] << " --slow-py3cc <slow-py3cc> SRC_PATH_X- SRC OUT" << Endl; + return 1; + } + + PyConfig cfg{}; + PyConfig_InitIsolatedConfig(&cfg); + cfg._install_importlib = 0; + Y_SCOPE_EXIT(&cfg) {PyConfig_Clear(&cfg);}; + + const char* slowPy3cc{argv[2]}; + TString srcpath{argv[3]}; + srcpath.pop_back(); + const TFsPath inPath{argv[4]}; + const char* outPath = argv[5]; + + const auto status = Py_InitializeFromConfig(&cfg); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + Y_SCOPE_EXIT() {Py_Finalize();}; + + TPyObject bytecode{Py_CompileString( + TFileInput{inPath}.ReadAll().c_str(), + srcpath.c_str(), + Py_file_input + )}; + if (!bytecode) { + int rc = runSlowPy3cc(slowPy3cc, argv[3], argv[4], argv[5]); + return rc; + } + + if (FILE* out = fopen(outPath, "wb")) { + PyMarshal_WriteObjectToFile(bytecode.Get(), out, Py_MARSHAL_VERSION); + fclose(out); + if (PyErr_Occurred()) { + Cerr << "Failed to marshal " << outPath << Endl; + PyErr_Print(); + return 1; + } + } else { + Cerr << "Failed to write " << outPath << ": " << std::error_code{errno, std::system_category()}.message() << Endl; + return 1; + } + + return 0; +} diff --git a/tools/py3cc/slow/bin/ya.make b/tools/py3cc/slow/bin/ya.make new file mode 100644 index 00000000000..c8c0c771214 --- /dev/null +++ b/tools/py3cc/slow/bin/ya.make @@ -0,0 +1,28 @@ +PY3_PROGRAM_BIN(py3cc) + +ENABLE(PYBUILD_NO_PYC) + +DISABLE(PYTHON_SQLITE3) + +PEERDIR( + library/python/runtime_py3 + library/python/runtime_py3/main +) + +NO_CHECK_IMPORTS() + +NO_PYTHON_INCLUDES() + +NO_PYTHON_COVERAGE() + +NO_IMPORT_TRACING() + +SRCDIR( + tools/py3cc/slow +) + +PY_SRCS( + MAIN main.py +) + +END() diff --git a/tools/py3cc/main.py b/tools/py3cc/slow/main.py index 996edb2bdda..996edb2bdda 100755 --- a/tools/py3cc/main.py +++ b/tools/py3cc/slow/main.py diff --git a/tools/py3cc/slow/ya.make b/tools/py3cc/slow/ya.make new file mode 100644 index 00000000000..283a03dbee1 --- /dev/null +++ b/tools/py3cc/slow/ya.make @@ -0,0 +1,11 @@ +IF (USE_PREBUILT_TOOLS AND NOT USE_PYTHON3_PREV) + INCLUDE(ya.make.prebuilt) +ENDIF() + +IF (NOT PREBUILT) + INCLUDE(bin/ya.make) +ENDIF() + +RECURSE( + bin +) diff --git a/tools/py3cc/ya.make b/tools/py3cc/ya.make index b6f5ce202eb..ea4ff598d05 100644 --- a/tools/py3cc/ya.make +++ b/tools/py3cc/ya.make @@ -1,4 +1,4 @@ -IF (USE_PREBUILT_TOOLS) +IF (USE_PREBUILT_TOOLS AND NOT USE_PYTHON3_PREV) INCLUDE(ya.make.prebuilt) ENDIF() @@ -8,4 +8,5 @@ ENDIF() RECURSE( bin + slow ) diff --git a/tools/ruff_linter/wrapper.py b/tools/ruff_linter/wrapper.py new file mode 100644 index 00000000000..f21fa737812 --- /dev/null +++ b/tools/ruff_linter/wrapper.py @@ -0,0 +1,157 @@ +import logging +import os +import subprocess +import time +import tomllib +from pathlib import Path + +from build.plugins.lib.test_const import RUFF_RESOURCE +from library.python.testing.custom_linter_util import linter_params, reporter +from library.python.testing.style import rules + +logger = logging.getLogger(__name__) + +FORMAT_SNIPPET_LINES_LIMIT = 100 + + +def get_ruff_bin(params) -> str: + ruff_root = params.global_resources[RUFF_RESOURCE] + return os.path.join(ruff_root, 'bin', 'ruff') + + +def check_extend_option_present(config: Path) -> bool: + with config.open('rb') as afile: + cfg = tomllib.load(afile) + if config.name == 'pyproject.toml' and cfg.get('tool', {}).get('ruff'): + return 'extend' in cfg['tool']['ruff'] + elif config.name == 'ruff.toml': + return 'extend' in cfg + raise RuntimeError(f'Unknown config type: {config.name}') + + +def run_ruff(ruff_bin: str, cmd_args: list[str], filename: str, config: Path) -> list[str]: + # XXX: `--no-cache` is important when we run ruff in source root and don't want to pollute arcadia + cmd = [ruff_bin, *cmd_args, '--no-cache', '--config', config, filename] + res = subprocess.run( + cmd, + capture_output=True, + encoding='utf8', + errors='replace', + env=dict(os.environ.copy(), RUFF_OUTPUT_FORMAT='concise', RAYON_NUM_THREADS='1'), + # When config is passed through `--config`, `exclude` starts searching from cwd + # so set cwd to config cwd to mimic behavior of autodiscovery. + # Note that it stops being accurate when `extend` is used. + # https://docs.astral.sh/ruff/configuration/#config-file-discovery + cwd=os.path.dirname(config), + ) + return res.stdout.splitlines(keepends=True) if res.returncode else [] + + +def process_file( + orig_filename: str, ruff_bin: str, orig_config: Path, source_root: str, check_format: bool, run_in_source_root: bool +) -> str: + logger.debug('Check %s with config %s', orig_filename, orig_config) + + file_path = os.path.relpath(orig_filename, source_root) + + if run_in_source_root: + filename = os.path.realpath(orig_filename) if os.path.islink(orig_filename) else orig_filename + config = orig_config.resolve() if orig_config.is_symlink() else orig_config + else: + filename = orig_filename + config = orig_config + + if check_format: + ruff_format_check_out = run_ruff(ruff_bin, ['format', '--diff'], filename, config) + if len(ruff_format_check_out) > FORMAT_SNIPPET_LINES_LIMIT: + ruff_format_check_out = ruff_format_check_out[:FORMAT_SNIPPET_LINES_LIMIT] + ruff_format_check_out.append('[truncated]...\n') + # first two lines are absolute file paths, replace with relative ones + if ruff_format_check_out: + ruff_format_check_out[0] = f'--- [[imp]]{file_path}[[rst]]\n' + ruff_format_check_out[1] = f'+++ [[imp]]{file_path}[[rst]]\n' + else: + ruff_format_check_out = [] + + ruff_check_out = run_ruff(ruff_bin, ['check', '-q'], filename, config) + # Every line starts with an absolute path to a file, replace with relative one + for idx, line in enumerate(ruff_check_out): + ruff_check_out[idx] = f'[[imp]]{file_path}[[rst]]:{line.split(':', 1)[-1]}' + + msg = '' + if ruff_format_check_out: + msg += '[[bad]]Formatting errors[[rst]]:\n' + msg += ''.join(ruff_format_check_out) + + if ruff_format_check_out and ruff_check_out: + msg += '\n' + + if ruff_check_out: + msg += '[[bad]]Linting errors[[rst]]:\n' + msg += ''.join(ruff_check_out) + + return msg + + +def main(): + params = linter_params.get_params() + + style_config_path = Path(params.configs[0]) + + # TODO: Ideally, to enable `extend` we first should move execution to build root (src files + configs) + # otherwise we risk allowing to steal from arcadia. To do that we need to mark modules 1st-party/3rd-party + # in pyproject.toml. + # UPD: it turned out to be more complicated for uservices-like projects because they use TOP_LEVEL / NAMESPACES + extend_option_present = check_extend_option_present(style_config_path) + + ruff_bin = get_ruff_bin(params) + + report = reporter.LintReport() + for file_name in params.files: + start_time = time.perf_counter() + + if extend_option_present: + elapsed = time.perf_counter() - start_time + report.add( + file_name, + reporter.LintStatus.FAIL, + "`extend` option in not supported in ruff config files for now. Modify your configs not to have it.", + elapsed=elapsed, + ) + continue + + skip_reason = rules.get_skip_reason(file_name, Path(file_name).read_text(), skip_links=False) + if skip_reason: + elapsed = time.perf_counter() - start_time + report.add( + file_name, + reporter.LintStatus.SKIPPED, + f"Style check is omitted: {skip_reason}", + elapsed=elapsed, + ) + continue + + error = process_file( + file_name, + ruff_bin, + style_config_path, + params.source_root, + params.extra_params.get('check_format') == 'yes', + params.extra_params.get('run_in_source_root') == 'yes', + ) + elapsed = time.perf_counter() - start_time + + if error: + rel_file_name = os.path.relpath(file_name, params.source_root) + message = f'Run [[imp]]ya style --ruff {rel_file_name}[[rst]] to fix format\n{error}' + status = reporter.LintStatus.FAIL + else: + message = '' + status = reporter.LintStatus.GOOD + report.add(file_name, status, message, elapsed=elapsed) + + report.dump(params.report_file) + + +if __name__ == '__main__': + main() |