diff options
author | zaverden <zaverden@yandex-team.com> | 2024-02-26 11:58:12 +0300 |
---|---|---|
committer | zaverden <zaverden@yandex-team.com> | 2024-02-26 12:16:26 +0300 |
commit | d180f9b302f71970d2e5cf0bf7cd26bdaa5aef12 (patch) | |
tree | 8a4b8a497a7296e4ff6977dd4dfa768b479aa647 | |
parent | 2c831425e3b8e6331c6ed07081cd0111db18bf9d (diff) | |
download | ydb-d180f9b302f71970d2e5cf0bf7cd26bdaa5aef12.tar.gz |
feat(conf): configure used tarballs by own pnpm-lock.yaml
b61ff371d6ad2f8c646f2a04e4a7d0fa178a6541
19 files changed, 183 insertions, 175 deletions
diff --git a/build/conf/proto.conf b/build/conf/proto.conf index b3fc5bc0a7..4d867c45df 100644 --- a/build/conf/proto.conf +++ b/build/conf/proto.conf @@ -759,6 +759,12 @@ multimodule PROTO_LIBRARY { # To include TS_PROTO user have to set INCLUDE_TAGS(TS_PROTO) in ya.make .INCLUDE_TAG=no .EPILOGUE=_TS_CONFIG_EPILOGUE + .PEERDIRSELF=TS_PREPARE_DEPS + } + + module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE { + .INCLUDE_TAG=no + .IGNORED=PEERDIR } module DESC_PROTO: _BARE_UNIT { diff --git a/build/conf/ts/node_modules.conf b/build/conf/ts/node_modules.conf index 13953e3c67..b50f6d8f15 100644 --- a/build/conf/ts/node_modules.conf +++ b/build/conf/ts/node_modules.conf @@ -1,70 +1,10 @@ PNPM_ROOT= PNPM_SCRIPT=$PNPM_ROOT/node_modules/pnpm/dist/pnpm.cjs NPM_CONTRIBS_PATH=contrib/typescript -# inputs list, just paths, deprecated (use _NODE_MODULES_INOUTS instead), used only for eslint/jest/hermione -_NODE_MODULES_INS= -# outputs list, just paths, deprecated (use _NODE_MODULES_INOUTS instead), used only for eslint/jest/hermione -_NODE_MODULES_OUTS= # combined input/outputs records as list of directives ${input;hide:<path>} ${output;hide:<path>}, used in builders _NODE_MODULES_INOUTS= _YATOOL_PREBUILDER_ARG= -# TOUCH_UNIT is required to create module identity file. -# We can "call" macro as `$_GET_NODE_MODULES_INS_OUTS(...)`. in this case we will get .CMD from it. -# This is the only way to process a variable data as an array. -# ${output;hide:_NODE_MODULES_OUTS} does not produce list of paths, but a single value (space-separeted paths) -_NODE_MODULES_CMD=$TOUCH_UNIT \ - && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS create-node-modules \ - $_GET_NODE_MODULES_INS_OUTS(IN $_NODE_MODULES_INS OUT $_NODE_MODULES_OUTS) \ - ${kv;hide:"pc magenta"} ${kv;hide:"p TS_NM"} - - -module _NODE_MODULES_BASE: _BARE_UNIT { - .CMD=_NODE_MODULES_CMD - # ignore SRCS macro, use TS_FILES instead of FILES - .ALIASES=SRCS=_NOOP_MACRO FILES=TS_FILES - # Propagates peers to related modules - .PEERDIR_POLICY=as_build_from - .NODE_TYPE=Bundle - - # TODO: remove this. YMAKE-1096 / FBP-1184 - _NEVERCACHE() - - # we have several modules in the same dir (.PEERDIRSELF=NODE_MODULES in BUILD) - # we need different names for module identity file - # .fake tells builder to not materialize it in results - SET(MODULE_SUFFIX .n_m.fake) - # .NODE_TYPE=Bundle is required for peers propagation, but it also affects - # how merging of pic/nopic graphs. Here we can override this merging behaviour - SET(MODULE_TYPE LIBRARY) - # define own tag - SET(MODULE_TAG NODE_MODULES) - # what modules it can PEERDIR to - SET(PEERDIR_TAGS TS TS_PROTO NPM_CONTRIBS) - # do not include it into "results" of graph - DISABLE(START_TARGET) - - # we read package.json and erm-packages.json during configuration - SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/pnpm-lock.yaml ${CURDIR}/package.json ${ARCADIA_ROOT}/$ERM_PACKAGES_PATH) - - PEERDIR($NPM_CONTRIBS_PATH) - # PEERDIR to the right version of nodejs and pnpm - _PEERDIR_TS_RESOURCE(nodejs pnpm) - - # run py logic - _NODE_MODULES_CONFIGURE() -} - -# called in on_node_modules_configure -macro _SET_NODE_MODULES_INS_OUTS(IN{input}[], OUT{output}[]) { - SET(_NODE_MODULES_INS $IN) - SET(_NODE_MODULES_OUTS $OUT) -} - -macro _GET_NODE_MODULES_INS_OUTS(IN{input}[], OUT{output}[]) { - .CMD=${input;hide:IN} ${output;hide:OUT} -} - ### @usage: NPM_CONTRIBS() # internal ### @@ -105,9 +45,52 @@ macro _FROM_NPM(TARBALL_URL, SKY_ID, INTEGRITY, INTEGRITY_ALGO, TARBALL_PATH) { } macro _TS_ADD_NODE_MODULES_FOR_BUILDER() { - # Provide downloaded dependencies in `/contrib/typescript/-` - PEERDIR($NPM_CONTRIBS_PATH) - # Calculate inputs and outputs of node_modules, fill `_NODE_MODULES_INOUTS` variable _NODE_MODULES_CONFIGURE() } + +_TARBALLS_STORE=__tarballs__ +_PREPARE_DEPS_INOUTS= +_PREPARE_DEPS_CMD=$TOUCH_UNIT \ + && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS prepare-deps \ + --tarballs-store $_TARBALLS_STORE \ + $_PREPARE_DEPS_INOUTS \ + ${kv;hide:"pc magenta"} ${kv;hide:"p TS_DEP"} + +# In case of no deps we need to create empty outputs for graph connectivity +_PREPARE_NO_DEPS_CMD=$TOUCH_UNIT \ + && $YMAKE_PYTHON ${input:"build/scripts/touch.py"} \ + $_PREPARE_DEPS_INOUTS \ + ${kv;hide:"pc magenta"} ${kv;hide:"p TS_NODEP"} + +module _PREPARE_DEPS_BASE: _BARE_UNIT { + .CMD=_PREPARE_DEPS_CMD + .IGNORED=SRCS FILES TS_FILES + # Propagates peers to related modules + .PEERDIR_POLICY=as_build_from + .NODE_TYPE=Bundle + .INCLUDE_TAG=no + + # we have several modules in the same dir (.PEERDIRSELF=TS_PREPARE_DEPS in BUILD) + # we need different names for module identity file + # .fake tells builder to not materialize it in results + SET(MODULE_SUFFIX .prepare_deps.fake) + # .NODE_TYPE=Bundle is required for peers propagation, but it also affects + # how merging of pic/nopic graphs. Here we can override this merging behaviour + SET(MODULE_TYPE LIBRARY) + # define own tag + SET(MODULE_TAG TS_PREPARE_DEPS) + # what modules it can PEERDIR to + SET(PEERDIR_TAGS TS_PREPARE_DEPS) + # do not include it into "results" of graph + DISABLE(START_TARGET) + + # we read pnpm-lock.yaml and package.json during configuration + SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/pnpm-lock.yaml ${CURDIR}/package.json) + + PEERDIR($NPM_CONTRIBS_PATH) + + _PREPARE_DEPS_CONFIGURE() +} + + diff --git a/build/conf/ts/ts_next.conf b/build/conf/ts/ts_next.conf index 0bf65e3e2d..58465b3a0d 100644 --- a/build/conf/ts/ts_next.conf +++ b/build/conf/ts/ts_next.conf @@ -27,6 +27,7 @@ multimodule TS_NEXT { module BUILD: _TS_BASE_UNIT { .CMD=TS_NEXT_CMD .EPILOGUE=_TS_CONFIG_EPILOGUE + .PEERDIRSELF=TS_PREPARE_DEPS # by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case) # but we have to set it to TS for include processor to work @@ -42,6 +43,10 @@ multimodule TS_NEXT { SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml ${CURDIR}/${TS_CONFIG_PATH}) _TS_ADD_NODE_MODULES_FOR_BUILDER() } + + module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE { + + } } macro TS_NEXT_CONFIG(Path) { diff --git a/build/conf/ts/ts_package.conf b/build/conf/ts/ts_package.conf index bad2085a62..135e41a39e 100644 --- a/build/conf/ts/ts_package.conf +++ b/build/conf/ts/ts_package.conf @@ -25,6 +25,7 @@ multimodule TS_PACKAGE { .CMD=TS_PACK .ALLOWED=TS_FILES .ALIASES=FILES=TS_FILES SRCS=TS_FILES + .PEERDIRSELF=TS_PREPARE_DEPS # by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case) # but we have to set it to TS for include processor to work @@ -33,4 +34,8 @@ multimodule TS_PACKAGE { SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml) _TS_ADD_NODE_MODULES_FOR_BUILDER() } + + module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE { + + } } diff --git a/build/conf/ts/ts_proto.conf b/build/conf/ts/ts_proto.conf index dc8e83e43a..115ec0f498 100644 --- a/build/conf/ts/ts_proto.conf +++ b/build/conf/ts/ts_proto.conf @@ -12,7 +12,7 @@ _TS_PROTO_IMPL_CMD=$TOUCH_UNIT \ ${kv;hide:"pc magenta"} ${kv;hide:"p TS_PRO"} - +### # internal module _TS_PROTO_IMPL: _TS_BASE_UNIT { .CMD=_TS_PROTO_IMPL_CMD .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_SKIFF CPP_PROTO_PLUGIN2 PY_PROTO_PLUGIN YMAPS_SPROTO RESOURCE @@ -39,6 +39,7 @@ module _TS_PROTO_IMPL: _TS_BASE_UNIT { _TS_ADD_NODE_MODULES_FOR_BUILDER() } +### @usage: _TS_PROTO_SRCS(path1 path2) # internal macro _TS_PROTO_SRCS(FILES...) { _SET_APPEND_WITH_DIRECTIVE(_TS_PROTO_SRCS_FILES input $FILES) }
\ No newline at end of file diff --git a/build/conf/ts/ts_tsc.conf b/build/conf/ts/ts_tsc.conf index bb7fb3bbef..cba05e44a8 100644 --- a/build/conf/ts/ts_tsc.conf +++ b/build/conf/ts/ts_tsc.conf @@ -21,6 +21,7 @@ multimodule TS_TSC { module BUILD: _TS_BASE_UNIT { .CMD=TS_TSC_CMD .EPILOGUE=_TS_CONFIG_EPILOGUE + .PEERDIRSELF=TS_PREPARE_DEPS # by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case) # but we have to set it to TS for include processor to work @@ -36,4 +37,8 @@ multimodule TS_TSC { SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml ${CURDIR}/${TS_CONFIG_PATH}) _TS_ADD_NODE_MODULES_FOR_BUILDER() } + + module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE { + + } } diff --git a/build/conf/ts/ts_vite.conf b/build/conf/ts/ts_vite.conf index 95e6ac8c62..04ee37227e 100644 --- a/build/conf/ts/ts_vite.conf +++ b/build/conf/ts/ts_vite.conf @@ -35,6 +35,7 @@ multimodule TS_VITE { module BUILD: _TS_BASE_UNIT { .CMD=TS_VITE_CMD .EPILOGUE=_TS_CONFIG_EPILOGUE + .PEERDIRSELF=TS_PREPARE_DEPS # by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case) # but we have to set it to TS for include processor to work @@ -50,4 +51,8 @@ multimodule TS_VITE { SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml ${CURDIR}/${TS_CONFIG_PATH}) _TS_ADD_NODE_MODULES_FOR_BUILDER() } + + module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE { + + } } diff --git a/build/conf/ts/ts_webpack.conf b/build/conf/ts/ts_webpack.conf index 6438147c4f..6e9e0571cf 100644 --- a/build/conf/ts/ts_webpack.conf +++ b/build/conf/ts/ts_webpack.conf @@ -35,6 +35,7 @@ multimodule TS_WEBPACK { module BUILD: _TS_BASE_UNIT { .CMD=TS_WEBPACK_CMD .EPILOGUE=_TS_CONFIG_EPILOGUE + .PEERDIRSELF=TS_PREPARE_DEPS # by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case) # but we have to set it to TS for include processor to work @@ -50,4 +51,8 @@ multimodule TS_WEBPACK { SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml ${CURDIR}/${TS_CONFIG_PATH}) _TS_ADD_NODE_MODULES_FOR_BUILDER() } + + module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE { + + } } diff --git a/build/plugins/lib/nots/package_manager/base/lockfile.py b/build/plugins/lib/nots/package_manager/base/lockfile.py index f13168b320..b6b9952602 100644 --- a/build/plugins/lib/nots/package_manager/base/lockfile.py +++ b/build/plugins/lib/nots/package_manager/base/lockfile.py @@ -9,16 +9,17 @@ class LockfilePackageMeta(object): Basic struct representing package meta from lockfile. """ - __slots__ = ("tarball_url", "sky_id", "integrity", "integrity_algorithm", "tarball_path") + __slots__ = ("key", "tarball_url", "sky_id", "integrity", "integrity_algorithm", "tarball_path") @staticmethod def from_str(s): return LockfilePackageMeta(*s.strip().split(" ")) - def __init__(self, tarball_url, sky_id, integrity, integrity_algorithm): + def __init__(self, key, tarball_url, sky_id, integrity, integrity_algorithm): # http://npm.yandex-team.ru/@scope%2fname/-/name-0.0.1.tgz parts = tarball_url.split("/") + self.key = key self.tarball_url = tarball_url self.sky_id = sky_id self.integrity = integrity diff --git a/build/plugins/lib/nots/package_manager/base/package_json.py b/build/plugins/lib/nots/package_manager/base/package_json.py index cc498b33f7..2438511800 100644 --- a/build/plugins/lib/nots/package_manager/base/package_json.py +++ b/build/plugins/lib/nots/package_manager/base/package_json.py @@ -130,10 +130,12 @@ class PackageJson(object): return None + # TODO: FBP-1254 + # def get_workspace_dep_spec_paths(self) -> list[tuple[str, str]]: def get_workspace_dep_spec_paths(self): """ Returns names and paths from specifiers of the defined workspace dependencies. - :rtype: list of (str, str) + :rtype: list[tuple[str, str]] """ spec_paths = [] schema = self.WORKSPACE_SCHEMA diff --git a/build/plugins/lib/nots/package_manager/base/package_manager.py b/build/plugins/lib/nots/package_manager/base/package_manager.py index 6b9faa56e8..1c7cedb662 100644 --- a/build/plugins/lib/nots/package_manager/base/package_manager.py +++ b/build/plugins/lib/nots/package_manager/base/package_manager.py @@ -133,8 +133,8 @@ class BasePackageManager(object): def _contrib_tarball_path(self, pkg): return os.path.join(self.contribs_path, pkg.tarball_path) - def _contrib_tarball_url(self, pkg): - return "file:" + self._contrib_tarball_path(pkg) + def _tarballs_store_path(self, pkg, store_path): + return os.path.join(self.module_path, store_path, pkg.tarball_path) def _get_default_options(self): return ["--registry", NPM_REGISTRY_URL] diff --git a/build/plugins/lib/nots/package_manager/pnpm/__init__.py b/build/plugins/lib/nots/package_manager/pnpm/__init__.py index af6de8e62a..0f50359c47 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/__init__.py +++ b/build/plugins/lib/nots/package_manager/pnpm/__init__.py @@ -1,11 +1,12 @@ from . import constants from .lockfile import PnpmLockfile from .package_manager import PnpmPackageManager -from .utils import build_ws_config_path +from .utils import build_ws_config_path, build_lockfile_path from .workspace import PnpmWorkspace __all__ = [ + "build_lockfile_path", "build_ws_config_path", "constants", "PnpmLockfile", diff --git a/build/plugins/lib/nots/package_manager/pnpm/constants.py b/build/plugins/lib/nots/package_manager/pnpm/constants.py index e84a78c55e..10ca9e9272 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/constants.py +++ b/build/plugins/lib/nots/package_manager/pnpm/constants.py @@ -1,2 +1,7 @@ PNPM_WS_FILENAME = "pnpm-workspace.yaml" PNPM_LOCKFILE_FILENAME = "pnpm-lock.yaml" + +# This is a name of intermediate file that is used in TS_PREPARE_DEPS. +# This file has a structure same to pnpm-lock.yaml, but all tarballs +# a set relative to the build root. +PNPM_PRE_LOCKFILE_FILENAME = "pre.pnpm-lock.yaml" diff --git a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py index eca1e4015b..46558861bd 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py +++ b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py @@ -10,21 +10,22 @@ from six import iteritems from ..base import PackageJson, BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError +LOCKFILE_VERSION = "lockfileVersion" + class PnpmLockfile(BaseLockfile): IMPORTER_KEYS = PackageJson.DEP_KEYS + ("specifiers",) def read(self): with io.open(self.path, "rb") as f: - self.data = yaml.load(f, Loader=yaml.CSafeLoader) + self.data = yaml.load(f, Loader=yaml.CSafeLoader) or {LOCKFILE_VERSION: "6.0"} - lockfileVersion = "lockfileVersion" - version_in_data = lockfileVersion in self.data + version_in_data = LOCKFILE_VERSION in self.data r = re.compile('^[56]\\.\\d$') - if not version_in_data or not r.match(str(self.data[lockfileVersion])): + if not version_in_data or not r.match(str(self.data[LOCKFILE_VERSION])): raise Exception( 'Error of project configuration: {} has lockfileVersion: {}. '.format( - self.path, self.data[lockfileVersion] if version_in_data else "<no-version>" + self.path, self.data[LOCKFILE_VERSION] if version_in_data else "<no-version>" ) + 'This version is not supported. Please, delete pnpm-lock.yaml and regenerate it using "ya tool nots --clean update-lockfile"' ) @@ -129,7 +130,7 @@ def _parse_package_meta(key, meta, allow_file_protocol=False): except LockfilePackageMetaInvalidError as e: raise TypeError("Invalid package meta for key {}, parse error: {}".format(key, e)) - return LockfilePackageMeta(tarball_url, sky_id, integrity, integrity_algorithm) + return LockfilePackageMeta(key, tarball_url, sky_id, integrity, integrity_algorithm) def _parse_tarball_url(tarball_url, allow_file_protocol): diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py index 3568dbff16..00151f4f51 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py +++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py @@ -1,10 +1,8 @@ import os import shutil -from six import iteritems - from .lockfile import PnpmLockfile -from .utils import build_lockfile_path, build_ws_config_path +from .utils import build_pre_lockfile_path, build_lockfile_path, build_ws_config_path from .workspace import PnpmWorkspace from ..base import BasePackageManager, PackageManagerError from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME @@ -82,69 +80,54 @@ class PnpmPackageManager(BasePackageManager): bundle_path=os.path.join(self.build_path, NODE_MODULES_WORKSPACE_BUNDLE_FILENAME), ) + # TODO: FBP-1254 + # def calc_prepare_deps_inouts(self, store_path: str, has_deps: bool) -> (list[str], list[str]): + def calc_prepare_deps_inouts(self, store_path, has_deps): + ins = [ + s_rooted(build_pj_path(self.module_path)), + s_rooted(build_lockfile_path(self.module_path)), + ] + outs = [ + b_rooted(build_ws_config_path(self.module_path)), + b_rooted(build_pre_lockfile_path(self.module_path)), + ] + + if has_deps: + for dep_path in self.get_local_peers_from_package_json(): + ins.append(b_rooted(build_ws_config_path(dep_path))) + ins.append(b_rooted(build_pre_lockfile_path(dep_path))) + + for pkg in self.extract_packages_meta_from_lockfiles([build_lockfile_path(self.sources_path)]): + ins.append(b_rooted(self._contrib_tarball_path(pkg))) + outs.append(b_rooted(self._tarballs_store_path(pkg, store_path))) + + return ins, outs + + # TODO: FBP-1254 + # def calc_node_modules_inouts(self, local_cli=False) -> (list[str], list[str]): def calc_node_modules_inouts(self, local_cli=False): """ Returns input and output paths for command that creates `node_modules` bundle. - Errors: errors caught while processing lockfiles + It relies on .PEERDIRSELF=TS_PREPARE_DEPS Inputs: - - source package.json and lockfile, - - built package.jsons of all deps, - - merged lockfiles and workspace configs of direct non-leave deps, - - tarballs. + - source package.json + - merged lockfiles and workspace configs of TS_PREPARE_DEPS Outputs: - - merged lockfile, - - generated workspace config, - - created node_modules bundle. - :rtype: (list of errors, list of str, list of str) + - created node_modules bundle """ - ins = [ - s_rooted(build_pj_path(self.module_path)), - ] + ins = [s_rooted(build_pj_path(self.module_path))] outs = [] pj = self.load_package_json_from_dir(self.sources_path) if pj.has_dependencies(): - ins.extend( - [ - s_rooted(build_lockfile_path(self.module_path)), - ] - ) - outs.extend( - [ - b_rooted(build_lockfile_path(self.module_path)), - b_rooted(build_ws_config_path(self.module_path)), - ] - ) + ins.append(b_rooted(build_pre_lockfile_path(self.module_path))) + ins.append(b_rooted(build_ws_config_path(self.module_path))) if not local_cli: - outs.extend([b_rooted(build_nm_bundle_path(self.module_path))]) - - # Source lockfiles are used only to get tarballs info. - src_lf_paths = [build_lockfile_path(self.sources_path)] - - for [dep_src_path, (_, depth)] in iteritems(pj.get_workspace_map(ignore_self=True)): - dep_mod_path = dep_src_path[len(self.sources_root) + 1 :] - # pnpm requires all package.jsons. - ins.append(b_rooted(build_pj_path(dep_mod_path))) - - dep_lf_src_path = build_lockfile_path(dep_src_path) - if not os.path.isfile(dep_lf_src_path): - # It is ok for leaves. - continue - src_lf_paths.append(dep_lf_src_path) + outs.append(b_rooted(build_nm_bundle_path(self.module_path))) + for dep_path in self.get_local_peers_from_package_json(): + ins.append(b_rooted(build_pj_path(dep_path))) - if depth == 1: - ins.append(b_rooted(build_ws_config_path(dep_mod_path))) - ins.append(b_rooted(build_lockfile_path(dep_mod_path))) - - errors = [] - try: - for pkg in self.extract_packages_meta_from_lockfiles(src_lf_paths): - ins.append(b_rooted(self._contrib_tarball_path(pkg))) - except Exception as e: - errors.append(e) - pass - - return errors, ins, outs + return ins, outs def extract_packages_meta_from_lockfiles(self, lf_paths): """ @@ -167,6 +150,13 @@ class PnpmPackageManager(BasePackageManager): raise PackageManagerError("Unable to process some lockfiles:\n{}".format("\n".join(errors))) def _prepare_workspace(self): + lf = self.load_lockfile(build_pre_lockfile_path(self.build_path)) + lf.update_tarball_resolutions(lambda p: "file:" + os.path.join(self.build_root, p.tarball_url)) + lf.write(build_lockfile_path(self.build_path)) + + return PnpmWorkspace.load(build_ws_config_path(self.build_path)) + + def build_workspace(self, tarballs_store): """ :rtype: PnpmWorkspace """ @@ -177,7 +167,7 @@ class PnpmPackageManager(BasePackageManager): dep_paths = ws.get_paths(ignore_self=True) self._build_merged_workspace_config(ws, dep_paths) - self._build_merged_lockfile(dep_paths) + self._build_merged_pre_lockfile(tarballs_store, dep_paths) return ws @@ -195,21 +185,21 @@ class PnpmPackageManager(BasePackageManager): return pj - def _build_merged_lockfile(self, dep_paths): + def _build_merged_pre_lockfile(self, tarballs_store, dep_paths): """ :type dep_paths: list of str :rtype: PnpmLockfile """ lf = self.load_lockfile_from_dir(self.sources_path) # Change to the output path for correct path calcs on merging. - lf.path = build_lockfile_path(self.build_path) + lf.path = build_pre_lockfile_path(self.build_path) + lf.update_tarball_resolutions(lambda p: self._tarballs_store_path(p, tarballs_store)) for dep_path in dep_paths: - lf_path = build_lockfile_path(dep_path) - if os.path.isfile(lf_path): - lf.merge(self.load_lockfile(lf_path)) + pre_lf_path = build_pre_lockfile_path(dep_path) + if os.path.isfile(pre_lf_path): + lf.merge(self.load_lockfile(pre_lf_path)) - lf.update_tarball_resolutions(lambda p: self._contrib_tarball_url(p)) lf.write() def _build_merged_workspace_config(self, ws, dep_paths): diff --git a/build/plugins/lib/nots/package_manager/pnpm/utils.py b/build/plugins/lib/nots/package_manager/pnpm/utils.py index 1fa4291b9d..9b6b9d80db 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/utils.py +++ b/build/plugins/lib/nots/package_manager/pnpm/utils.py @@ -1,6 +1,10 @@ import os -from .constants import PNPM_LOCKFILE_FILENAME, PNPM_WS_FILENAME +from .constants import PNPM_PRE_LOCKFILE_FILENAME, PNPM_LOCKFILE_FILENAME, PNPM_WS_FILENAME + + +def build_pre_lockfile_path(p): + return os.path.join(p, PNPM_PRE_LOCKFILE_FILENAME) def build_lockfile_path(p): diff --git a/build/plugins/lib/nots/package_manager/pnpm/workspace.py b/build/plugins/lib/nots/package_manager/pnpm/workspace.py index e596e20a18..5639ae1c20 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/workspace.py +++ b/build/plugins/lib/nots/package_manager/pnpm/workspace.py @@ -20,7 +20,8 @@ class PnpmWorkspace(object): def read(self): with open(self.path) as f: - self.packages = set(yaml.load(f, Loader=yaml.CSafeLoader).get("packages", [])) + parsed = yaml.load(f, Loader=yaml.CSafeLoader) or {} + self.packages = set(parsed.get("packages", [])) def write(self, path=None): if not path: diff --git a/build/plugins/lib/nots/typescript/tests/test_ts_config.py b/build/plugins/lib/nots/typescript/tests/test_ts_config.py index cf67ca5ff9..43ae2d6738 100644 --- a/build/plugins/lib/nots/typescript/tests/test_ts_config.py +++ b/build/plugins/lib/nots/typescript/tests/test_ts_config.py @@ -48,11 +48,7 @@ def test_ts_config_validate_empty(): def test_ts_config_declaration_with_dir(): cfg = TsConfig(path="/tsconfig.json") cfg.data = { - "compilerOptions": { - "rootDir": "./src", - "declaration": True, - "declarationDir": "some/dir" - }, + "compilerOptions": {"rootDir": "./src", "declaration": True, "declarationDir": "some/dir"}, } cfg.validate(use_outdir=False) @@ -61,10 +57,7 @@ def test_ts_config_declaration_with_dir(): def test_ts_config_declaration_without_dir(): cfg = TsConfig(path="/tsconfig.json") cfg.data = { - "compilerOptions": { - "rootDir": "./src", - "declaration": True - }, + "compilerOptions": {"rootDir": "./src", "declaration": True}, } # When outDir should not be used we got the error @@ -79,11 +72,7 @@ def test_ts_config_declaration_without_dir(): def test_ts_config_declaration_with_outdir(): cfg = TsConfig(path="/tsconfig.json") cfg.data = { - "compilerOptions": { - "rootDir": "./src", - "outDir": "some/dir", - "declaration": True - }, + "compilerOptions": {"rootDir": "./src", "outDir": "some/dir", "declaration": True}, } # When we allow outDir it will be enought to set it diff --git a/build/plugins/nots.py b/build/plugins/nots.py index 6e99f5c10b..cc4561bb0b 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -422,10 +422,6 @@ def _add_test(unit, test_type, test_files, deps=None, test_record=None, test_cwd if test_record: full_test_record.update(test_record) - for k, v in full_test_record.items(): - if not isinstance(v, str): - logger.warn(k, "expected 'str', got:", type(v)) - data = ytest.dump_test(unit, full_test_record) if data: unit.set_property(["DART_DATA", data]) @@ -480,6 +476,23 @@ def _select_matching_version(erm_json, resource_name, range_str, dep_is_required @_with_report_configure_error +def on_prepare_deps_configure(unit): + pm = _create_pm(unit) + pj = pm.load_package_json_from_dir(pm.sources_path) + has_deps = pj.has_dependencies() + ins, outs = pm.calc_prepare_deps_inouts(unit.get("_TARBALLS_STORE"), has_deps) + + if pj.has_dependencies(): + unit.onpeerdir(pm.get_local_peers_from_package_json()) + __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("input", ["hide"], sorted(ins))) + __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", ["hide"], sorted(outs))) + + else: + __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", [], sorted(outs))) + unit.set(["_PREPARE_DEPS_CMD", "$_PREPARE_NO_DEPS_CMD"]) + + +@_with_report_configure_error def on_node_modules_configure(unit): pm = _create_pm(unit) pj = pm.load_package_json_from_dir(pm.sources_path) @@ -487,21 +500,11 @@ def on_node_modules_configure(unit): if pj.has_dependencies(): unit.onpeerdir(pm.get_local_peers_from_package_json()) local_cli = unit.get("TS_LOCAL_CLI") == "yes" - errors, ins, outs = pm.calc_node_modules_inouts(local_cli) - - if errors: - ymake.report_configure_error( - "There are some issues with lockfiles.\n" - + "Please contact support (https://nda.ya.ru/t/sNoSFsO76ygSXL),\n" - + "providing following details:\n" - + "\n---\n".join([str(err) for err in errors]) - ) - else: - unit.on_set_node_modules_ins_outs(["IN"] + sorted(ins) + ["OUT"] + sorted(outs)) + ins, outs = pm.calc_node_modules_inouts(local_cli) - __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("input", ["hide"], sorted(ins))) - if not unit.get("TS_TEST_FOR"): - __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("output", ["hide"], sorted(outs))) + __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("input", ["hide"], sorted(ins))) + if not unit.get("TS_TEST_FOR"): + __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("output", ["hide"], sorted(outs))) if pj.get_use_prebuilder(): lf = pm.load_lockfile_from_dir(pm.sources_path) @@ -526,10 +529,6 @@ def on_node_modules_configure(unit): ] ) - else: - # default "noop" command - unit.set(["_NODE_MODULES_CMD", "$TOUCH_UNIT"]) - @_with_report_configure_error def on_ts_test_for_configure(unit, test_runner, default_config, node_modules_filename): |