diff options
author | alevitskii <alevitskii@yandex-team.com> | 2024-09-12 09:41:59 +0300 |
---|---|---|
committer | alevitskii <alevitskii@yandex-team.com> | 2024-09-12 09:53:26 +0300 |
commit | f3fc9bb5ecb70b4b2a340d935ea06c92307da3b1 (patch) | |
tree | a705eae6ae543a0f75fe061412fea9e91d59e842 /build | |
parent | 7bd81dc2f1ea38b950587f937d2f886cf011e60e (diff) | |
download | ydb-f3fc9bb5ecb70b4b2a340d935ea06c92307da3b1.tar.gz |
Python linting macroses
Init
93a5052a4192d230f52b4b27c41c9c77bf8d5880
Diffstat (limited to 'build')
-rw-r--r-- | build/conf/python.conf | 54 | ||||
-rw-r--r-- | build/plugins/_dart_fields.py | 128 | ||||
-rw-r--r-- | build/plugins/pybuild.py | 59 | ||||
-rw-r--r-- | build/plugins/ytest.py | 62 |
4 files changed, 224 insertions, 79 deletions
diff --git a/build/conf/python.conf b/build/conf/python.conf index 29c1f712fa..30b1fe2d04 100644 --- a/build/conf/python.conf +++ b/build/conf/python.conf @@ -179,6 +179,24 @@ macro NO_YMAKE_PYTHON3() { SET(YMAKE_PYTHON3_PEERDIR) } +FLAKE_EXTRA_PARAMS= +FLAKE_MIGRATIONS_CONFIG= +FLAKE_CONFIG_FILES=build/config/tests/flake8/flake8.conf $FLAKE_MIGRATIONS_CONFIG +when ($DISABLE_FLAKE8_MIGRATIONS == "yes") { + FLAKE_EXTRA_PARAMS="DISABLE_FLAKE8_MIGRATIONS=yes" +} +otherwise { + FLAKE_MIGRATIONS_CONFIG=build/rules/flake8/migrations.yaml +} + +when ($FLAKE8_FILE_PROCESSING_TIME == "") { + FLAKE8_FILE_PROCESSING_TIME=1.5 +} + +when ($BLACK_FILE_PROCESSING_TIME == "") { + BLACK_FILE_PROCESSING_TIME=1.5 +} + # tag:python-specific tag:deprecated tag:internal module _PY2_PROGRAM: _BASE_PY_PROGRAM { SET(MODULE_LANG PY2) @@ -186,6 +204,7 @@ module _PY2_PROGRAM: _BASE_PY_PROGRAM { # Looks like we cannot avoid copy-paste util ymake supports multiple inheritance # We need to attach coverage.extractor to every py_program target, except pytest targets ADD_YTEST($MODULE_PREFIX$REALPRJNAME coverage.extractor) + STYLE_PY2_FLAKE8() } # tag:python-specific tag:deprecated @@ -247,25 +266,39 @@ multimodule PY3_PROGRAM { } # tag:python-specific tag:test -STYLE_PYTHON_VALUE=no -STYLE_PYTHON_PYPROJECT_VALUE= ### @usage: STYLE_PYTHON([pyproject]) ### ### Check python3 sources for style issues using black. +BLACK_CONFIG_FILES= macro STYLE_PYTHON(pyproject...) { - SET(STYLE_PYTHON_VALUE yes) - SET(STYLE_PYTHON_PYPROJECT_VALUE ${pyproject}) + BLACK_CONFIG_FILES=$pyproject build/config/tests/py_style/config.toml + _ADD_PY_LINTER_CHECK(NAME black LINTER tools/black_linter/black_linter FILE_PROCESSING_TIME $BLACK_FILE_PROCESSING_TIME CONFIGS $BLACK_CONFIG_FILES) } # tag:python-specific tag:test -STYLE_RUFF_VALUE=no -RUFF_CONFIG_PATHS_FILE=${ARCADIA_ROOT}/build/config/tests/ruff/ruff_config_paths.json ### @usage: STYLE_RUFF() ### ### Check python3 sources for style issues using ruff. +RUFF_CONFIG_PATHS_FILE=build/config/tests/ruff/ruff_config_paths.json macro STYLE_RUFF() { - SET(STYLE_RUFF_VALUE yes) - SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${RUFF_CONFIG_PATHS_FILE}) + SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/${RUFF_CONFIG_PATHS_FILE}) + _ADD_PY_LINTER_CHECK(NAME ruff LINTER tools/ruff_linter/bin/ruff_linter GLOBAL_RESOURCES build/external_resources/ruff CONFIGS $RUFF_CONFIG_PATHS_FILE) +} + +# tag:python-specific tag:test +### @usage: STYLE_FLAKE8() +### +### Check python3 sources for style issues using flake8. +macro STYLE_FLAKE8() { + _ADD_PY_LINTER_CHECK(NAME flake8 LINTER tools/flake8_linter/flake8_linter GLOBAL_RESOURCES build/external_resources/flake8_py3 FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $FLAKE_CONFIG_FILES EXTRA_PARAMS $FLAKE_EXTRA_PARAMS) +} + +# tag:python-specific tag:test +### @usage: STYLE_PY2_FLAKE8() +### +### Check python3 sources for style issues using flake8. +macro STYLE_PY2_FLAKE8() { + _ADD_PY_LINTER_CHECK(NAME py2_flake8 LINTER tools/flake8_linter/flake8_linter GLOBAL_RESOURCES build/external_resources/flake8_py2 FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $FLAKE_CONFIG_FILES EXTRA_PARAMS $FLAKE_EXTRA_PARAMS) } # tag:python-specific tag:test @@ -302,6 +335,7 @@ module PYTEST_BIN: _BASE_PYTEST { .DEFAULT_NAME_GENERATOR=FullPath .ARGS_PARSER=Base SETUP_PYTEST_BIN() + STYLE_PY2_FLAKE8() } # tag:python-specific tag:test @@ -344,6 +378,7 @@ module PY3TEST_BIN: _BASE_PY3_PROGRAM { .ARGS_PARSER=Base SET(MODULE_LANG PY3) SETUP_PYTEST_BIN() + STYLE_FLAKE8() PEERDIR+=library/python/pytest } @@ -565,6 +600,7 @@ module PY2_LIBRARY: _LIBRARY { } SET(MODULE_LANG PY2) + STYLE_PY2_FLAKE8() ADD_CLANG_TIDY() when ($TIDY_ENABLED == "yes") { _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json @@ -611,6 +647,7 @@ module PY3_LIBRARY: _LIBRARY { } SET(MODULE_LANG PY3) + STYLE_FLAKE8() ADD_CLANG_TIDY() when ($TIDY_ENABLED == "yes") { _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json @@ -749,6 +786,7 @@ module _BASE_PY3_PROGRAM: _BASE_PROGRAM { module PY3_PROGRAM_BIN: _BASE_PY3_PROGRAM { # Look's like we cannot avoid copy-paste util ymake supports multiple inheritance # We need to attach coverage.extractor to every py_program target, except pytest targets + STYLE_FLAKE8() ADD_YTEST($MODULE_PREFIX$REALPRJNAME coverage.extractor) } diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py index 4676499399..07ff9ce0f2 100644 --- a/build/plugins/_dart_fields.py +++ b/build/plugins/_dart_fields.py @@ -22,6 +22,9 @@ CANON_OUTPUT_STORAGE = 'canondata_storage' KTLINT_CURRENT_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint/.editorconfig" KTLINT_OLD_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint_old/.editorconfig" +ARCADIA_ROOT = '${ARCADIA_ROOT}/' +SOURCE_ROOT_SHORT = '$S/' + class DartValueError(ValueError): pass @@ -256,6 +259,15 @@ def _get_ts_test_data_dirs(unit): ) +@_common.lazy +def get_linter_configs(unit, config_paths): + rel_config_path = _common.rootrel_arc_src(config_paths, unit) + arc_config_path = unit.resolve_arc_path(rel_config_path) + abs_config_path = unit.resolve(arc_config_path) + with open(abs_config_path, 'r') as fd: + return list(json.load(fd).values()) + + class AndroidApkTestActivity: KEY = 'ANDROID_APK_TEST_ACTIVITY' @@ -360,10 +372,10 @@ class CustomDependencies: @classmethod def depends_with_linter(cls, unit, flat_args, spec_args): - deps = [] - _, linter = flat_args - deps.append(os.path.dirname(linter)) - deps += spec_args.get('DEPENDS', []) + linter = Linter.value(unit, flat_args, spec_args)[Linter.KEY] + deps = spec_args.get('DEPENDS', []) + [os.path.dirname(linter)] + for dep in deps: + unit.ondepends(dep) return {cls.KEY: " ".join(deps)} @classmethod @@ -478,7 +490,7 @@ class JavaClasspathCmdType: unit.path(), java_cp_arg_type ) ) - raise DartValueError + raise DartValueError() return {cls.KEY: java_cp_arg_type} @@ -528,9 +540,69 @@ class KtlintBinary: return {cls.KEY: value} +class Linter: + KEY = 'LINTER' + + @classmethod + def value(cls, unit, flat_args, spec_args): + return {cls.KEY: spec_args['LINTER'][0]} + + +class LintConfigs: + KEY = 'LINT-CONFIGS' + + @classmethod + def value(cls, unit, flat_args, spec_args): + resolved_configs = [] + configs = spec_args.get('CONFIGS', []) + for cfg in configs: + filename = unit.resolve(SOURCE_ROOT_SHORT + cfg) + if not os.path.exists(filename): + message = 'Configuration file {} is not found'.format(filename) + raise DartValueError(message) + resolved_configs.append(cfg) + if os.path.splitext(filename)[-1] == '.json': + cfgs = get_linter_configs(unit, cfg) + for c in cfgs: + filename = unit.resolve(SOURCE_ROOT_SHORT + c) + if not os.path.exists(filename): + message = 'Configuration file {} is not found'.format(filename) + raise DartValueError(message) + resolved_configs.append(c) + return {cls.KEY: serialize_list(resolved_configs)} + + +class LintExtraParams: + KEY = 'LINT-EXTRA-PARAMS' + + @classmethod + def from_macro_args(cls, unit, flat_args, spec_args): + extra_params = spec_args.get('EXTRA_PARAMS', []) + for arg in extra_params: + if '=' not in arg: + message = 'Wrong EXTRA_PARAMS value: "{}". Values must have format "name=value".'.format(arg) + raise DartValueError(message) + return {cls.KEY: serialize_list(extra_params)} + + class LintFileProcessingTime: KEY = 'LINT-FILE-PROCESSING-TIME' + @classmethod + def from_macro_args(cls, unit, flat_args, spec_args): + return {cls.KEY: spec_args.get('FILE_PROCESSING_TIME', [''])[0]} + + +class LintName: + KEY = 'LINT-NAME' + + @classmethod + def value(cls, unit, flat_args, spec_args): + lint_name = spec_args['NAME'][0] + if lint_name in ('flake8', 'py2_flake8') and (unit.get('DISABLE_FLAKE8') or 'no') == 'yes': + raise DartValueError('Flake8 linting is disabled by `DISABLE_FLAKE8`') + return {cls.KEY: lint_name} + class ModuleLang: KEY = 'MODULE_LANG' @@ -841,9 +913,8 @@ class TestData: props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE')) if error_mgs: - # TODO move error reporting out of field classes ymake.report_configure_error(error_mgs) - raise DartValueError + raise DartValueError() for prop in props: if prop['type'] == 'file': test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia')) @@ -949,7 +1020,8 @@ class TestedProjectName: class TestFiles: KEY = 'TEST-FILES' - # TODO remove FILES, see DEVTOOLS-7052 + # TODO remove FILES, see DEVTOOLS-7052, currently it's required + # https://a.yandex-team.ru/arcadia/devtools/ya/test/dartfile/__init__.py?rev=r14292146#L10 KEY2 = 'FILES' @classmethod @@ -993,32 +1065,55 @@ class TestFiles: @classmethod def test_srcs(cls, unit, flat_args, spec_args): test_files = get_values_list(unit, 'TEST_SRCS_VALUE') - return {cls.KEY: serialize_list(test_files)} + value = serialize_list(test_files) + return {cls.KEY: value, cls.KEY2: value} @classmethod def ts_test_srcs(cls, unit, flat_args, spec_args): test_files = get_values_list(unit, "_TS_TEST_SRCS_VALUE") test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files) - return {cls.KEY: serialize_list(test_files)} + value = serialize_list(test_files) + return {cls.KEY: value, cls.KEY2: value} @classmethod def ts_input_files(cls, unit, flat_args, spec_args): typecheck_files = get_values_list(unit, "TS_INPUT_FILES") test_files = [_common.resolve_common_const(f) for f in typecheck_files] - return {cls.KEY: serialize_list(test_files)} + value = serialize_list(test_files) + return {cls.KEY: value, cls.KEY2: value} @classmethod def ts_lint_srcs(cls, unit, flat_args, spec_args): test_files = get_values_list(unit, "_TS_LINT_SRCS_VALUE") test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files) - return {cls.KEY: serialize_list(test_files)} + value = serialize_list(test_files) + return {cls.KEY: value, cls.KEY2: value} @classmethod def stylesheets(cls, unit, flat_args, spec_args): test_files = get_values_list(unit, "_TS_STYLELINT_FILES") test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files) + value = serialize_list(test_files) + return {cls.KEY: value, cls.KEY2: value} - return {cls.KEY: serialize_list(test_files)} + @classmethod + def py_linter_files(cls, unit, flat_args, spec_args): + files = unit.get('PY_LINTER_FILES') + if not files: + raise DartValueError() + files = json.loads(files) + test_files = [] + for path in files: + if path.startswith(ARCADIA_ROOT): + test_files.append(path.replace(ARCADIA_ROOT, SOURCE_ROOT_SHORT, 1)) + elif path.startswith(SOURCE_ROOT_SHORT): + test_files.append(path) + if not test_files: + lint_name = LintName.value(unit, flat_args, spec_args)[LintName.KEY] + message = 'No files to lint for {}'.format(lint_name) + raise DartValueError(message) + test_files = serialize_list(test_files) + return {cls.KEY: test_files, cls.KEY2: test_files} class TestEnv: @@ -1097,6 +1192,10 @@ class TestName: test_name = os.path.basename(os.path.join(unit.path(), unit.filename()).replace(".pkg", "")) return {cls.KEY: os.path.splitext(test_name)[0]} + @classmethod + def name_from_macro_args(cls, unit, flat_args, spec_args): + return {cls.KEY: spec_args['NAME'][0]} + class TestPartition: KEY = 'TEST_PARTITION' @@ -1181,9 +1280,8 @@ class SystemProperties: def value(cls, unit, flat_args, spec_args): props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE')) if error_mgs: - # TODO move error reporting out of field classes ymake.report_configure_error(error_mgs) - raise DartValueError + raise DartValueError() props = base64.b64encode(six.ensure_binary(json.dumps(props))) return {cls.KEY: props} diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index 231107b0a4..bca88ebedc 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -183,62 +183,9 @@ def add_python_lint_checks(unit, py_ver, files): if files and no_lint_value not in ("none", "none_internal"): resolved_files = get_resolved_files() if resolved_files: - flake8_cfg = 'build/config/tests/flake8/flake8.conf' - migrations_cfg = 'build/rules/flake8/migrations.yaml' - resource = "build/external_resources/flake8_py{}".format(py_ver) - lint_name = "py2_flake8" if py_ver == 2 else "flake8" - params = [lint_name, "tools/flake8_linter/flake8_linter"] - params += ["FILES"] + resolved_files - params += ["GLOBAL_RESOURCES", resource] - params += [ - "FILE_PROCESSING_TIME", - unit.get("FLAKE8_FILE_PROCESSING_TIME") or DEFAULT_FLAKE8_FILE_PROCESSING_TIME, - ] - - extra_params = [] - if unit.get("DISABLE_FLAKE8_MIGRATIONS") == "yes": - extra_params.append("DISABLE_FLAKE8_MIGRATIONS=yes") - config_files = [flake8_cfg, ''] - else: - config_files = [flake8_cfg, migrations_cfg] - params += ["CONFIGS"] + config_files - - if extra_params: - params += ["EXTRA_PARAMS"] + extra_params - unit.on_add_linter_check(params) - - # ruff related stuff - if unit.get('STYLE_RUFF_VALUE') == 'yes': - if no_lint_value in ("none", "none_internal"): - ymake.report_configure_error( - 'NO_LINT() and STYLE_RUFF() can\'t be enabled both at the same time', - ) - - resolved_files = get_resolved_files() - if resolved_files: - resource = "build/external_resources/ruff" - params = ["ruff", "tools/ruff_linter/bin/ruff_linter"] - params += ["FILES"] + resolved_files - params += ["GLOBAL_RESOURCES", resource] - configs = [ - rootrel_arc_src(unit.get('RUFF_CONFIG_PATHS_FILE'), unit), - 'build/config/tests/ruff/ruff.toml', - ] + get_ruff_configs(unit) - params += ['CONFIGS'] + configs - unit.on_add_linter_check(params) - - if files and unit.get('STYLE_PYTHON_VALUE') == 'yes' and is_py3(unit): - resolved_files = get_resolved_files() - if resolved_files: - black_cfg = unit.get('STYLE_PYTHON_PYPROJECT_VALUE') or 'build/config/tests/py_style/config.toml' - params = ['black', 'tools/black_linter/black_linter'] - params += ['FILES'] + resolved_files - params += ['CONFIGS', black_cfg] - params += [ - "FILE_PROCESSING_TIME", - unit.get("BLACK_FILE_PROCESSING_TIME") or DEFAULT_BLACK_FILE_PROCESSING_TIME, - ] - unit.on_add_linter_check(params) + # repeated PY_SRCS, TEST_SCRS+PY_SRCS + collected = json.loads(unit.get('PY_LINTER_FILES')) if unit.get('PY_LINTER_FILES') else [] + unit.set(['PY_LINTER_FILES', json.dumps(resolved_files + collected)]) def is_py3(unit): diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index aa41dff565..46ee8d2599 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -947,6 +947,68 @@ def onsetup_run_python(unit): unit.ondepends('contrib/tools/python') +@_common.lazy +def get_linter_configs(unit, config_paths): + rel_config_path = _common.rootrel_arc_src(config_paths, unit) + arc_config_path = unit.resolve_arc_path(rel_config_path) + abs_config_path = unit.resolve(arc_config_path) + with open(abs_config_path, 'r') as fd: + return list(json.load(fd).values()) + + +@df.with_fields( + ( + df.LintName.value, + df.TestFiles.py_linter_files, + df.LintConfigs.value, + df.LintExtraParams.from_macro_args, + df.TestName.name_from_macro_args, + df.TestedProjectName.unit_name, + df.SourceFolderPath.normalized, + df.TestEnv.value, + df.UseArcadiaPython.value, + df.LintFileProcessingTime.from_macro_args, + df.Linter.value, + df.CustomDependencies.depends_with_linter, + ) +) +def on_add_py_linter_check(fields, unit, *args): + if unit.get("TIDY") == "yes": + return + + no_lint_value = _common.get_no_lint_value(unit) + if no_lint_value in ("none", "none_internal"): + return + + unlimited = -1 + keywords = { + "NAME": 1, + "LINTER": 1, + "DEPENDS": unlimited, + "FILES": unlimited, + "CONFIGS": unlimited, + "GLOBAL_RESOURCES": unlimited, + "FILE_PROCESSING_TIME": 1, + "EXTRA_PARAMS": unlimited, + } + _, spec_args = _common.sort_by_keywords(keywords, args) + + global_resources = spec_args.get('GLOBAL_RESOURCES', []) + for resource in global_resources: + unit.onpeerdir(resource) + try: + dart_record = create_dart_record(fields, unit, (), spec_args) + except df.DartValueError as e: + if msg := str(e): + unit.message(['WARN', msg]) + return + dart_record[df.ScriptRelPath.KEY] = 'custom_lint' + + data = dump_test(unit, dart_record) + if data: + unit.set_property(["DART_DATA", data]) + + def on_add_linter_check(unit, *args): if unit.get("TIDY") == "yes": return |