diff options
author | khoden <khoden@yandex-team.com> | 2023-09-08 15:29:00 +0300 |
---|---|---|
committer | khoden <khoden@yandex-team.com> | 2023-09-08 15:56:50 +0300 |
commit | 1327369eb8af90ffcedd1ea4f256c374c52097e1 (patch) | |
tree | a9c6154771777732ad4d1e568ca0de9b38dfab12 /build/plugins | |
parent | 2a4344f50e5d190be68407dca1cada777cb04c45 (diff) | |
download | ydb-1327369eb8af90ffcedd1ea4f256c374c52097e1.tar.gz |
Изменить логику дискаверинга исходников для TS_*
### Принцип работы:
1. В переменную с помощью `_GLOB` записываем все файлы (*.ts *.tsx *.js *.jsx *.json) проекта, кроме дефолтного списка исключений (типа `node_modules`, `dist/build`, `a.yaml`, `ya.make`, `package.json`, `pnpm-lock.yaml` и подобные)
2. Вызываем плагин (`_TS_CONFIGURE`), где удаляем из этого списка файлы, не покрываемые `tsconfig.json`
3. Получившееся значение используем в качестве источника инпутов для таргета
Diffstat (limited to 'build/plugins')
-rw-r--r-- | build/plugins/lib/nots/typescript/tests/test_ts_glob.py | 75 | ||||
-rw-r--r-- | build/plugins/lib/nots/typescript/tests/ya.make | 1 | ||||
-rw-r--r-- | build/plugins/lib/nots/typescript/ts_config.py | 23 | ||||
-rw-r--r-- | build/plugins/lib/nots/typescript/ts_glob.py | 77 | ||||
-rw-r--r-- | build/plugins/lib/nots/typescript/ya.make | 1 | ||||
-rw-r--r-- | build/plugins/nots.py | 23 |
6 files changed, 198 insertions, 2 deletions
diff --git a/build/plugins/lib/nots/typescript/tests/test_ts_glob.py b/build/plugins/lib/nots/typescript/tests/test_ts_glob.py new file mode 100644 index 0000000000..9d38a8d80f --- /dev/null +++ b/build/plugins/lib/nots/typescript/tests/test_ts_glob.py @@ -0,0 +1,75 @@ +from build.plugins.lib.nots.typescript.ts_glob import ts_glob, TsGlobConfig + + +class TestTsGlobIncluding: + ts_glob_config = TsGlobConfig( + root_dir="src", out_dir="build", include=["src/module_a/**/*", "src/module_b/**/*", "src/module_x"] + ) + + def test_dir_include(self): + # arrange + all_files = [ + "src/module_x/index.ts", + ] + + # act + arrange + assert ts_glob(self.ts_glob_config, all_files) == [ + "src/module_x/index.ts", + ] + + def test_deep_include(self): + # arrange + all_files = [ + "src/module_a/index.ts", + "src/module_b/index.ts", + "src/module_c/index.ts", + ] + + # act + arrange + assert ts_glob(self.ts_glob_config, all_files) == [ + "src/module_a/index.ts", + "src/module_b/index.ts", + ] + + +class TestTsGlobExcluding: + ts_glob_config = TsGlobConfig(root_dir="src", out_dir="build", include=["src/**/*"]) + + def test_only_in_root_dir(self): + # arrange + all_files = [ + "CHANGELOG.md", + "fake-src/one-more-src.ts", + "src/index.ts", + ] + + # act + assert + assert ts_glob(self.ts_glob_config, all_files) == ["src/index.ts"] + + def test_exclude_out_dir(self): + # arrange + all_files = ["build/index.js"] + + # act + assert + assert ts_glob(self.ts_glob_config, all_files) == [] + + def test_exclude_out_dir_none(self): + # arrange + all_files = ["build/index.js"] + ts_glob_config = TsGlobConfig(root_dir=".", out_dir=None) + + # act + assert + assert ts_glob(ts_glob_config, all_files) == ["build/index.js"] + + def test_complex(self): + # arrange + all_files = [ + "CHANGELOG.md", + "fake-src/one-more-src.ts", + "src/baz.ts", + "src/index.ts", + "src/required_file.ts", + ] + + # act + assert + assert ts_glob(self.ts_glob_config, all_files) == ["src/baz.ts", "src/index.ts", "src/required_file.ts"] diff --git a/build/plugins/lib/nots/typescript/tests/ya.make b/build/plugins/lib/nots/typescript/tests/ya.make index 44798138bc..2e038f6c96 100644 --- a/build/plugins/lib/nots/typescript/tests/ya.make +++ b/build/plugins/lib/nots/typescript/tests/ya.make @@ -4,6 +4,7 @@ OWNER(g:frontend-build-platform) TEST_SRCS( ts_config.py + test_ts_glob.py ) PEERDIR( diff --git a/build/plugins/lib/nots/typescript/ts_config.py b/build/plugins/lib/nots/typescript/ts_config.py index c54121a9d1..b4ad9c3d3f 100644 --- a/build/plugins/lib/nots/typescript/ts_config.py +++ b/build/plugins/lib/nots/typescript/ts_config.py @@ -1,9 +1,9 @@ import copy -import os import json +import os from .ts_errors import TsError, TsValidationError - +from .ts_glob import ts_glob, TsGlobConfig from ..package_manager.base import utils DEFAULT_TS_CONFIG_FILE = "tsconfig.json" @@ -249,3 +249,22 @@ class TsConfig(object): with open(path, "w") as f: json.dump(self.data, f, indent=indent) + + def filter_files(self, all_files): + # type: (list[str]) -> list[str] + """ + Filters all the files by the rules from this tsconig.json. The result will be used as input entries in `ya make`. + + Known limits: + + - `files` nots supported, use `include` (see `self.validate()`) + - `exclude` not implemented, because `tsc` still uses "excluded" files as declaration files (for typing and referencing) + """ + + ts_glob_config = TsGlobConfig( + root_dir=self.compiler_option("rootDir"), + out_dir=self.compiler_option("outDir"), + include=self.data.get("include"), + ) + + return ts_glob(ts_glob_config, all_files) diff --git a/build/plugins/lib/nots/typescript/ts_glob.py b/build/plugins/lib/nots/typescript/ts_glob.py new file mode 100644 index 0000000000..b81e8774a5 --- /dev/null +++ b/build/plugins/lib/nots/typescript/ts_glob.py @@ -0,0 +1,77 @@ +import fnmatch +import os.path + + +class TsGlobConfig: + def __init__(self, root_dir, out_dir=None, include=None): + # type: (TsGlobConfig, str, str, list[str], list[str]) -> None + + self.root_dir = root_dir # Required + self.out_dir = out_dir + + self.include = include or ["**/*"] + + +def __path_to_match_rule(path): + # type: (str) -> str + + # already a rule + + # convert "**/*" to "*" (python compatible with fnmatch) + if path.endswith('**/*'): + return path[:-3] # /**/* -> /* + + if path.endswith("*") or ('*' in path or '?' in path): + return path + + # special cases + if path == ".": + return "*" + + # filename + root, ext = os.path.splitext(path) + if ext: + return path + + # dirname ? + return os.path.join(path, '*') + + +def __filter_files(files, path_or_rule): + # type: (set[str], str) -> set[str] + + rule = __path_to_match_rule(path_or_rule) + + result = set() + for path in files: + py_rule = __path_to_match_rule(rule) + if path == rule or fnmatch.fnmatch(path, py_rule): + result.add(path) + + return result + + +def ts_glob(glob_config, all_files): + # type: (TsGlobConfig, list[str]) -> list[str] + + result = set(all_files) + + # only in `root_dir` + result &= __filter_files(result, glob_config.root_dir) + + # only listed by `include` option + include_only = set() + for include_path in glob_config.include: + include_only |= __filter_files(result, include_path) + + result &= include_only # keep only intersection (common in both sets) + + skip_files = set() + + # exclude out_dir + if glob_config.out_dir: + skip_files |= __filter_files(result, glob_config.out_dir) + + result -= skip_files # keep only differences (the elements in `result` that not exist in `skip_files`) + + return sorted(result) diff --git a/build/plugins/lib/nots/typescript/ya.make b/build/plugins/lib/nots/typescript/ya.make index 8847f9bbd3..ac31196472 100644 --- a/build/plugins/lib/nots/typescript/ya.make +++ b/build/plugins/lib/nots/typescript/ya.make @@ -5,6 +5,7 @@ OWNER(g:frontend-build-platform) PY_SRCS( __init__.py ts_errors.py + ts_glob.py ts_config.py ) diff --git a/build/plugins/nots.py b/build/plugins/nots.py index 740b042376..f6363dfd6f 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -200,6 +200,29 @@ def on_ts_configure(unit, tsconfig_path): unit.set(["TS_CONFIG_PRESERVE_JSX", to_yesno(tsconfig.compiler_option("jsx") == "preserve")]) _setup_eslint(unit) + _filter_inputs_by_rules_from_tsconfig(unit, tsconfig) + + +def __strip_prefix(prefix, line): + # type: (str, str) -> str + if line.startswith(prefix): + prefix_len = len(prefix) + return line[prefix_len:] + + return line + + +def _filter_inputs_by_rules_from_tsconfig(unit, tsconfig): + """ + Reduce file list from the TS_GLOB_FILES variable following tsconfig.json rules + """ + mod_dir = unit.get("MODDIR") + target_path = "${ARCADIA_ROOT}/" + mod_dir + "/" + + all_files = [__strip_prefix(target_path, f) for f in unit.get("TS_GLOB_FILES").split(" ")] + filtered_files = tsconfig.filter_files(all_files) + + unit.set(["TS_GLOB_FILES", ' '.join([target_path + f for f in filtered_files])]) def _get_ts_test_data_dirs(unit): |