aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorzaverden <zaverden@yandex-team.com>2024-02-26 11:58:12 +0300
committerzaverden <zaverden@yandex-team.com>2024-02-26 12:16:26 +0300
commitd180f9b302f71970d2e5cf0bf7cd26bdaa5aef12 (patch)
tree8a4b8a497a7296e4ff6977dd4dfa768b479aa647
parent2c831425e3b8e6331c6ed07081cd0111db18bf9d (diff)
downloadydb-d180f9b302f71970d2e5cf0bf7cd26bdaa5aef12.tar.gz
feat(conf): configure used tarballs by own pnpm-lock.yaml
b61ff371d6ad2f8c646f2a04e4a7d0fa178a6541
-rw-r--r--build/conf/proto.conf6
-rw-r--r--build/conf/ts/node_modules.conf109
-rw-r--r--build/conf/ts/ts_next.conf5
-rw-r--r--build/conf/ts/ts_package.conf5
-rw-r--r--build/conf/ts/ts_proto.conf3
-rw-r--r--build/conf/ts/ts_tsc.conf5
-rw-r--r--build/conf/ts/ts_vite.conf5
-rw-r--r--build/conf/ts/ts_webpack.conf5
-rw-r--r--build/plugins/lib/nots/package_manager/base/lockfile.py5
-rw-r--r--build/plugins/lib/nots/package_manager/base/package_json.py4
-rw-r--r--build/plugins/lib/nots/package_manager/base/package_manager.py4
-rw-r--r--build/plugins/lib/nots/package_manager/pnpm/__init__.py3
-rw-r--r--build/plugins/lib/nots/package_manager/pnpm/constants.py5
-rw-r--r--build/plugins/lib/nots/package_manager/pnpm/lockfile.py13
-rw-r--r--build/plugins/lib/nots/package_manager/pnpm/package_manager.py112
-rw-r--r--build/plugins/lib/nots/package_manager/pnpm/utils.py6
-rw-r--r--build/plugins/lib/nots/package_manager/pnpm/workspace.py3
-rw-r--r--build/plugins/lib/nots/typescript/tests/test_ts_config.py17
-rw-r--r--build/plugins/nots.py43
19 files changed, 183 insertions, 175 deletions
diff --git a/build/conf/proto.conf b/build/conf/proto.conf
index b3fc5bc0a7..4d867c45df 100644
--- a/build/conf/proto.conf
+++ b/build/conf/proto.conf
@@ -759,6 +759,12 @@ multimodule PROTO_LIBRARY {
# To include TS_PROTO user have to set INCLUDE_TAGS(TS_PROTO) in ya.make
.INCLUDE_TAG=no
.EPILOGUE=_TS_CONFIG_EPILOGUE
+ .PEERDIRSELF=TS_PREPARE_DEPS
+ }
+
+ module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
+ .INCLUDE_TAG=no
+ .IGNORED=PEERDIR
}
module DESC_PROTO: _BARE_UNIT {
diff --git a/build/conf/ts/node_modules.conf b/build/conf/ts/node_modules.conf
index 13953e3c67..b50f6d8f15 100644
--- a/build/conf/ts/node_modules.conf
+++ b/build/conf/ts/node_modules.conf
@@ -1,70 +1,10 @@
PNPM_ROOT=
PNPM_SCRIPT=$PNPM_ROOT/node_modules/pnpm/dist/pnpm.cjs
NPM_CONTRIBS_PATH=contrib/typescript
-# inputs list, just paths, deprecated (use _NODE_MODULES_INOUTS instead), used only for eslint/jest/hermione
-_NODE_MODULES_INS=
-# outputs list, just paths, deprecated (use _NODE_MODULES_INOUTS instead), used only for eslint/jest/hermione
-_NODE_MODULES_OUTS=
# combined input/outputs records as list of directives ${input;hide:<path>} ${output;hide:<path>}, used in builders
_NODE_MODULES_INOUTS=
_YATOOL_PREBUILDER_ARG=
-# TOUCH_UNIT is required to create module identity file.
-# We can "call" macro as `$_GET_NODE_MODULES_INS_OUTS(...)`. in this case we will get .CMD from it.
-# This is the only way to process a variable data as an array.
-# ${output;hide:_NODE_MODULES_OUTS} does not produce list of paths, but a single value (space-separeted paths)
-_NODE_MODULES_CMD=$TOUCH_UNIT \
- && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS create-node-modules \
- $_GET_NODE_MODULES_INS_OUTS(IN $_NODE_MODULES_INS OUT $_NODE_MODULES_OUTS) \
- ${kv;hide:"pc magenta"} ${kv;hide:"p TS_NM"}
-
-
-module _NODE_MODULES_BASE: _BARE_UNIT {
- .CMD=_NODE_MODULES_CMD
- # ignore SRCS macro, use TS_FILES instead of FILES
- .ALIASES=SRCS=_NOOP_MACRO FILES=TS_FILES
- # Propagates peers to related modules
- .PEERDIR_POLICY=as_build_from
- .NODE_TYPE=Bundle
-
- # TODO: remove this. YMAKE-1096 / FBP-1184
- _NEVERCACHE()
-
- # we have several modules in the same dir (.PEERDIRSELF=NODE_MODULES in BUILD)
- # we need different names for module identity file
- # .fake tells builder to not materialize it in results
- SET(MODULE_SUFFIX .n_m.fake)
- # .NODE_TYPE=Bundle is required for peers propagation, but it also affects
- # how merging of pic/nopic graphs. Here we can override this merging behaviour
- SET(MODULE_TYPE LIBRARY)
- # define own tag
- SET(MODULE_TAG NODE_MODULES)
- # what modules it can PEERDIR to
- SET(PEERDIR_TAGS TS TS_PROTO NPM_CONTRIBS)
- # do not include it into "results" of graph
- DISABLE(START_TARGET)
-
- # we read package.json and erm-packages.json during configuration
- SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/pnpm-lock.yaml ${CURDIR}/package.json ${ARCADIA_ROOT}/$ERM_PACKAGES_PATH)
-
- PEERDIR($NPM_CONTRIBS_PATH)
- # PEERDIR to the right version of nodejs and pnpm
- _PEERDIR_TS_RESOURCE(nodejs pnpm)
-
- # run py logic
- _NODE_MODULES_CONFIGURE()
-}
-
-# called in on_node_modules_configure
-macro _SET_NODE_MODULES_INS_OUTS(IN{input}[], OUT{output}[]) {
- SET(_NODE_MODULES_INS $IN)
- SET(_NODE_MODULES_OUTS $OUT)
-}
-
-macro _GET_NODE_MODULES_INS_OUTS(IN{input}[], OUT{output}[]) {
- .CMD=${input;hide:IN} ${output;hide:OUT}
-}
-
### @usage: NPM_CONTRIBS() # internal
###
@@ -105,9 +45,52 @@ macro _FROM_NPM(TARBALL_URL, SKY_ID, INTEGRITY, INTEGRITY_ALGO, TARBALL_PATH) {
}
macro _TS_ADD_NODE_MODULES_FOR_BUILDER() {
- # Provide downloaded dependencies in `/contrib/typescript/-`
- PEERDIR($NPM_CONTRIBS_PATH)
-
# Calculate inputs and outputs of node_modules, fill `_NODE_MODULES_INOUTS` variable
_NODE_MODULES_CONFIGURE()
}
+
+_TARBALLS_STORE=__tarballs__
+_PREPARE_DEPS_INOUTS=
+_PREPARE_DEPS_CMD=$TOUCH_UNIT \
+ && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS prepare-deps \
+ --tarballs-store $_TARBALLS_STORE \
+ $_PREPARE_DEPS_INOUTS \
+ ${kv;hide:"pc magenta"} ${kv;hide:"p TS_DEP"}
+
+# In case of no deps we need to create empty outputs for graph connectivity
+_PREPARE_NO_DEPS_CMD=$TOUCH_UNIT \
+ && $YMAKE_PYTHON ${input:"build/scripts/touch.py"} \
+ $_PREPARE_DEPS_INOUTS \
+ ${kv;hide:"pc magenta"} ${kv;hide:"p TS_NODEP"}
+
+module _PREPARE_DEPS_BASE: _BARE_UNIT {
+ .CMD=_PREPARE_DEPS_CMD
+ .IGNORED=SRCS FILES TS_FILES
+ # Propagates peers to related modules
+ .PEERDIR_POLICY=as_build_from
+ .NODE_TYPE=Bundle
+ .INCLUDE_TAG=no
+
+ # we have several modules in the same dir (.PEERDIRSELF=TS_PREPARE_DEPS in BUILD)
+ # we need different names for module identity file
+ # .fake tells builder to not materialize it in results
+ SET(MODULE_SUFFIX .prepare_deps.fake)
+ # .NODE_TYPE=Bundle is required for peers propagation, but it also affects
+ # how merging of pic/nopic graphs. Here we can override this merging behaviour
+ SET(MODULE_TYPE LIBRARY)
+ # define own tag
+ SET(MODULE_TAG TS_PREPARE_DEPS)
+ # what modules it can PEERDIR to
+ SET(PEERDIR_TAGS TS_PREPARE_DEPS)
+ # do not include it into "results" of graph
+ DISABLE(START_TARGET)
+
+ # we read pnpm-lock.yaml and package.json during configuration
+ SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/pnpm-lock.yaml ${CURDIR}/package.json)
+
+ PEERDIR($NPM_CONTRIBS_PATH)
+
+ _PREPARE_DEPS_CONFIGURE()
+}
+
+
diff --git a/build/conf/ts/ts_next.conf b/build/conf/ts/ts_next.conf
index 0bf65e3e2d..58465b3a0d 100644
--- a/build/conf/ts/ts_next.conf
+++ b/build/conf/ts/ts_next.conf
@@ -27,6 +27,7 @@ multimodule TS_NEXT {
module BUILD: _TS_BASE_UNIT {
.CMD=TS_NEXT_CMD
.EPILOGUE=_TS_CONFIG_EPILOGUE
+ .PEERDIRSELF=TS_PREPARE_DEPS
# by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case)
# but we have to set it to TS for include processor to work
@@ -42,6 +43,10 @@ multimodule TS_NEXT {
SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml ${CURDIR}/${TS_CONFIG_PATH})
_TS_ADD_NODE_MODULES_FOR_BUILDER()
}
+
+ module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
+
+ }
}
macro TS_NEXT_CONFIG(Path) {
diff --git a/build/conf/ts/ts_package.conf b/build/conf/ts/ts_package.conf
index bad2085a62..135e41a39e 100644
--- a/build/conf/ts/ts_package.conf
+++ b/build/conf/ts/ts_package.conf
@@ -25,6 +25,7 @@ multimodule TS_PACKAGE {
.CMD=TS_PACK
.ALLOWED=TS_FILES
.ALIASES=FILES=TS_FILES SRCS=TS_FILES
+ .PEERDIRSELF=TS_PREPARE_DEPS
# by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case)
# but we have to set it to TS for include processor to work
@@ -33,4 +34,8 @@ multimodule TS_PACKAGE {
SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml)
_TS_ADD_NODE_MODULES_FOR_BUILDER()
}
+
+ module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
+
+ }
}
diff --git a/build/conf/ts/ts_proto.conf b/build/conf/ts/ts_proto.conf
index dc8e83e43a..115ec0f498 100644
--- a/build/conf/ts/ts_proto.conf
+++ b/build/conf/ts/ts_proto.conf
@@ -12,7 +12,7 @@ _TS_PROTO_IMPL_CMD=$TOUCH_UNIT \
${kv;hide:"pc magenta"} ${kv;hide:"p TS_PRO"}
-
+### # internal
module _TS_PROTO_IMPL: _TS_BASE_UNIT {
.CMD=_TS_PROTO_IMPL_CMD
.IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_SKIFF CPP_PROTO_PLUGIN2 PY_PROTO_PLUGIN YMAPS_SPROTO RESOURCE
@@ -39,6 +39,7 @@ module _TS_PROTO_IMPL: _TS_BASE_UNIT {
_TS_ADD_NODE_MODULES_FOR_BUILDER()
}
+### @usage: _TS_PROTO_SRCS(path1 path2) # internal
macro _TS_PROTO_SRCS(FILES...) {
_SET_APPEND_WITH_DIRECTIVE(_TS_PROTO_SRCS_FILES input $FILES)
} \ No newline at end of file
diff --git a/build/conf/ts/ts_tsc.conf b/build/conf/ts/ts_tsc.conf
index bb7fb3bbef..cba05e44a8 100644
--- a/build/conf/ts/ts_tsc.conf
+++ b/build/conf/ts/ts_tsc.conf
@@ -21,6 +21,7 @@ multimodule TS_TSC {
module BUILD: _TS_BASE_UNIT {
.CMD=TS_TSC_CMD
.EPILOGUE=_TS_CONFIG_EPILOGUE
+ .PEERDIRSELF=TS_PREPARE_DEPS
# by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case)
# but we have to set it to TS for include processor to work
@@ -36,4 +37,8 @@ multimodule TS_TSC {
SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml ${CURDIR}/${TS_CONFIG_PATH})
_TS_ADD_NODE_MODULES_FOR_BUILDER()
}
+
+ module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
+
+ }
}
diff --git a/build/conf/ts/ts_vite.conf b/build/conf/ts/ts_vite.conf
index 95e6ac8c62..04ee37227e 100644
--- a/build/conf/ts/ts_vite.conf
+++ b/build/conf/ts/ts_vite.conf
@@ -35,6 +35,7 @@ multimodule TS_VITE {
module BUILD: _TS_BASE_UNIT {
.CMD=TS_VITE_CMD
.EPILOGUE=_TS_CONFIG_EPILOGUE
+ .PEERDIRSELF=TS_PREPARE_DEPS
# by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case)
# but we have to set it to TS for include processor to work
@@ -50,4 +51,8 @@ multimodule TS_VITE {
SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml ${CURDIR}/${TS_CONFIG_PATH})
_TS_ADD_NODE_MODULES_FOR_BUILDER()
}
+
+ module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
+
+ }
}
diff --git a/build/conf/ts/ts_webpack.conf b/build/conf/ts/ts_webpack.conf
index 6438147c4f..6e9e0571cf 100644
--- a/build/conf/ts/ts_webpack.conf
+++ b/build/conf/ts/ts_webpack.conf
@@ -35,6 +35,7 @@ multimodule TS_WEBPACK {
module BUILD: _TS_BASE_UNIT {
.CMD=TS_WEBPACK_CMD
.EPILOGUE=_TS_CONFIG_EPILOGUE
+ .PEERDIRSELF=TS_PREPARE_DEPS
# by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case)
# but we have to set it to TS for include processor to work
@@ -50,4 +51,8 @@ multimodule TS_WEBPACK {
SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/package.json ${CURDIR}/pnpm-lock.yaml ${CURDIR}/${TS_CONFIG_PATH})
_TS_ADD_NODE_MODULES_FOR_BUILDER()
}
+
+ module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
+
+ }
}
diff --git a/build/plugins/lib/nots/package_manager/base/lockfile.py b/build/plugins/lib/nots/package_manager/base/lockfile.py
index f13168b320..b6b9952602 100644
--- a/build/plugins/lib/nots/package_manager/base/lockfile.py
+++ b/build/plugins/lib/nots/package_manager/base/lockfile.py
@@ -9,16 +9,17 @@ class LockfilePackageMeta(object):
Basic struct representing package meta from lockfile.
"""
- __slots__ = ("tarball_url", "sky_id", "integrity", "integrity_algorithm", "tarball_path")
+ __slots__ = ("key", "tarball_url", "sky_id", "integrity", "integrity_algorithm", "tarball_path")
@staticmethod
def from_str(s):
return LockfilePackageMeta(*s.strip().split(" "))
- def __init__(self, tarball_url, sky_id, integrity, integrity_algorithm):
+ def __init__(self, key, tarball_url, sky_id, integrity, integrity_algorithm):
# http://npm.yandex-team.ru/@scope%2fname/-/name-0.0.1.tgz
parts = tarball_url.split("/")
+ self.key = key
self.tarball_url = tarball_url
self.sky_id = sky_id
self.integrity = integrity
diff --git a/build/plugins/lib/nots/package_manager/base/package_json.py b/build/plugins/lib/nots/package_manager/base/package_json.py
index cc498b33f7..2438511800 100644
--- a/build/plugins/lib/nots/package_manager/base/package_json.py
+++ b/build/plugins/lib/nots/package_manager/base/package_json.py
@@ -130,10 +130,12 @@ class PackageJson(object):
return None
+ # TODO: FBP-1254
+ # def get_workspace_dep_spec_paths(self) -> list[tuple[str, str]]:
def get_workspace_dep_spec_paths(self):
"""
Returns names and paths from specifiers of the defined workspace dependencies.
- :rtype: list of (str, str)
+ :rtype: list[tuple[str, str]]
"""
spec_paths = []
schema = self.WORKSPACE_SCHEMA
diff --git a/build/plugins/lib/nots/package_manager/base/package_manager.py b/build/plugins/lib/nots/package_manager/base/package_manager.py
index 6b9faa56e8..1c7cedb662 100644
--- a/build/plugins/lib/nots/package_manager/base/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/base/package_manager.py
@@ -133,8 +133,8 @@ class BasePackageManager(object):
def _contrib_tarball_path(self, pkg):
return os.path.join(self.contribs_path, pkg.tarball_path)
- def _contrib_tarball_url(self, pkg):
- return "file:" + self._contrib_tarball_path(pkg)
+ def _tarballs_store_path(self, pkg, store_path):
+ return os.path.join(self.module_path, store_path, pkg.tarball_path)
def _get_default_options(self):
return ["--registry", NPM_REGISTRY_URL]
diff --git a/build/plugins/lib/nots/package_manager/pnpm/__init__.py b/build/plugins/lib/nots/package_manager/pnpm/__init__.py
index af6de8e62a..0f50359c47 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/__init__.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/__init__.py
@@ -1,11 +1,12 @@
from . import constants
from .lockfile import PnpmLockfile
from .package_manager import PnpmPackageManager
-from .utils import build_ws_config_path
+from .utils import build_ws_config_path, build_lockfile_path
from .workspace import PnpmWorkspace
__all__ = [
+ "build_lockfile_path",
"build_ws_config_path",
"constants",
"PnpmLockfile",
diff --git a/build/plugins/lib/nots/package_manager/pnpm/constants.py b/build/plugins/lib/nots/package_manager/pnpm/constants.py
index e84a78c55e..10ca9e9272 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/constants.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/constants.py
@@ -1,2 +1,7 @@
PNPM_WS_FILENAME = "pnpm-workspace.yaml"
PNPM_LOCKFILE_FILENAME = "pnpm-lock.yaml"
+
+# This is a name of intermediate file that is used in TS_PREPARE_DEPS.
+# This file has a structure same to pnpm-lock.yaml, but all tarballs
+# a set relative to the build root.
+PNPM_PRE_LOCKFILE_FILENAME = "pre.pnpm-lock.yaml"
diff --git a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
index eca1e4015b..46558861bd 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
@@ -10,21 +10,22 @@ from six import iteritems
from ..base import PackageJson, BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError
+LOCKFILE_VERSION = "lockfileVersion"
+
class PnpmLockfile(BaseLockfile):
IMPORTER_KEYS = PackageJson.DEP_KEYS + ("specifiers",)
def read(self):
with io.open(self.path, "rb") as f:
- self.data = yaml.load(f, Loader=yaml.CSafeLoader)
+ self.data = yaml.load(f, Loader=yaml.CSafeLoader) or {LOCKFILE_VERSION: "6.0"}
- lockfileVersion = "lockfileVersion"
- version_in_data = lockfileVersion in self.data
+ version_in_data = LOCKFILE_VERSION in self.data
r = re.compile('^[56]\\.\\d$')
- if not version_in_data or not r.match(str(self.data[lockfileVersion])):
+ if not version_in_data or not r.match(str(self.data[LOCKFILE_VERSION])):
raise Exception(
'Error of project configuration: {} has lockfileVersion: {}. '.format(
- self.path, self.data[lockfileVersion] if version_in_data else "<no-version>"
+ self.path, self.data[LOCKFILE_VERSION] if version_in_data else "<no-version>"
)
+ 'This version is not supported. Please, delete pnpm-lock.yaml and regenerate it using "ya tool nots --clean update-lockfile"'
)
@@ -129,7 +130,7 @@ def _parse_package_meta(key, meta, allow_file_protocol=False):
except LockfilePackageMetaInvalidError as e:
raise TypeError("Invalid package meta for key {}, parse error: {}".format(key, e))
- return LockfilePackageMeta(tarball_url, sky_id, integrity, integrity_algorithm)
+ return LockfilePackageMeta(key, tarball_url, sky_id, integrity, integrity_algorithm)
def _parse_tarball_url(tarball_url, allow_file_protocol):
diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
index 3568dbff16..00151f4f51 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
@@ -1,10 +1,8 @@
import os
import shutil
-from six import iteritems
-
from .lockfile import PnpmLockfile
-from .utils import build_lockfile_path, build_ws_config_path
+from .utils import build_pre_lockfile_path, build_lockfile_path, build_ws_config_path
from .workspace import PnpmWorkspace
from ..base import BasePackageManager, PackageManagerError
from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME
@@ -82,69 +80,54 @@ class PnpmPackageManager(BasePackageManager):
bundle_path=os.path.join(self.build_path, NODE_MODULES_WORKSPACE_BUNDLE_FILENAME),
)
+ # TODO: FBP-1254
+ # def calc_prepare_deps_inouts(self, store_path: str, has_deps: bool) -> (list[str], list[str]):
+ def calc_prepare_deps_inouts(self, store_path, has_deps):
+ ins = [
+ s_rooted(build_pj_path(self.module_path)),
+ s_rooted(build_lockfile_path(self.module_path)),
+ ]
+ outs = [
+ b_rooted(build_ws_config_path(self.module_path)),
+ b_rooted(build_pre_lockfile_path(self.module_path)),
+ ]
+
+ if has_deps:
+ for dep_path in self.get_local_peers_from_package_json():
+ ins.append(b_rooted(build_ws_config_path(dep_path)))
+ ins.append(b_rooted(build_pre_lockfile_path(dep_path)))
+
+ for pkg in self.extract_packages_meta_from_lockfiles([build_lockfile_path(self.sources_path)]):
+ ins.append(b_rooted(self._contrib_tarball_path(pkg)))
+ outs.append(b_rooted(self._tarballs_store_path(pkg, store_path)))
+
+ return ins, outs
+
+ # TODO: FBP-1254
+ # def calc_node_modules_inouts(self, local_cli=False) -> (list[str], list[str]):
def calc_node_modules_inouts(self, local_cli=False):
"""
Returns input and output paths for command that creates `node_modules` bundle.
- Errors: errors caught while processing lockfiles
+ It relies on .PEERDIRSELF=TS_PREPARE_DEPS
Inputs:
- - source package.json and lockfile,
- - built package.jsons of all deps,
- - merged lockfiles and workspace configs of direct non-leave deps,
- - tarballs.
+ - source package.json
+ - merged lockfiles and workspace configs of TS_PREPARE_DEPS
Outputs:
- - merged lockfile,
- - generated workspace config,
- - created node_modules bundle.
- :rtype: (list of errors, list of str, list of str)
+ - created node_modules bundle
"""
- ins = [
- s_rooted(build_pj_path(self.module_path)),
- ]
+ ins = [s_rooted(build_pj_path(self.module_path))]
outs = []
pj = self.load_package_json_from_dir(self.sources_path)
if pj.has_dependencies():
- ins.extend(
- [
- s_rooted(build_lockfile_path(self.module_path)),
- ]
- )
- outs.extend(
- [
- b_rooted(build_lockfile_path(self.module_path)),
- b_rooted(build_ws_config_path(self.module_path)),
- ]
- )
+ ins.append(b_rooted(build_pre_lockfile_path(self.module_path)))
+ ins.append(b_rooted(build_ws_config_path(self.module_path)))
if not local_cli:
- outs.extend([b_rooted(build_nm_bundle_path(self.module_path))])
-
- # Source lockfiles are used only to get tarballs info.
- src_lf_paths = [build_lockfile_path(self.sources_path)]
-
- for [dep_src_path, (_, depth)] in iteritems(pj.get_workspace_map(ignore_self=True)):
- dep_mod_path = dep_src_path[len(self.sources_root) + 1 :]
- # pnpm requires all package.jsons.
- ins.append(b_rooted(build_pj_path(dep_mod_path)))
-
- dep_lf_src_path = build_lockfile_path(dep_src_path)
- if not os.path.isfile(dep_lf_src_path):
- # It is ok for leaves.
- continue
- src_lf_paths.append(dep_lf_src_path)
+ outs.append(b_rooted(build_nm_bundle_path(self.module_path)))
+ for dep_path in self.get_local_peers_from_package_json():
+ ins.append(b_rooted(build_pj_path(dep_path)))
- if depth == 1:
- ins.append(b_rooted(build_ws_config_path(dep_mod_path)))
- ins.append(b_rooted(build_lockfile_path(dep_mod_path)))
-
- errors = []
- try:
- for pkg in self.extract_packages_meta_from_lockfiles(src_lf_paths):
- ins.append(b_rooted(self._contrib_tarball_path(pkg)))
- except Exception as e:
- errors.append(e)
- pass
-
- return errors, ins, outs
+ return ins, outs
def extract_packages_meta_from_lockfiles(self, lf_paths):
"""
@@ -167,6 +150,13 @@ class PnpmPackageManager(BasePackageManager):
raise PackageManagerError("Unable to process some lockfiles:\n{}".format("\n".join(errors)))
def _prepare_workspace(self):
+ lf = self.load_lockfile(build_pre_lockfile_path(self.build_path))
+ lf.update_tarball_resolutions(lambda p: "file:" + os.path.join(self.build_root, p.tarball_url))
+ lf.write(build_lockfile_path(self.build_path))
+
+ return PnpmWorkspace.load(build_ws_config_path(self.build_path))
+
+ def build_workspace(self, tarballs_store):
"""
:rtype: PnpmWorkspace
"""
@@ -177,7 +167,7 @@ class PnpmPackageManager(BasePackageManager):
dep_paths = ws.get_paths(ignore_self=True)
self._build_merged_workspace_config(ws, dep_paths)
- self._build_merged_lockfile(dep_paths)
+ self._build_merged_pre_lockfile(tarballs_store, dep_paths)
return ws
@@ -195,21 +185,21 @@ class PnpmPackageManager(BasePackageManager):
return pj
- def _build_merged_lockfile(self, dep_paths):
+ def _build_merged_pre_lockfile(self, tarballs_store, dep_paths):
"""
:type dep_paths: list of str
:rtype: PnpmLockfile
"""
lf = self.load_lockfile_from_dir(self.sources_path)
# Change to the output path for correct path calcs on merging.
- lf.path = build_lockfile_path(self.build_path)
+ lf.path = build_pre_lockfile_path(self.build_path)
+ lf.update_tarball_resolutions(lambda p: self._tarballs_store_path(p, tarballs_store))
for dep_path in dep_paths:
- lf_path = build_lockfile_path(dep_path)
- if os.path.isfile(lf_path):
- lf.merge(self.load_lockfile(lf_path))
+ pre_lf_path = build_pre_lockfile_path(dep_path)
+ if os.path.isfile(pre_lf_path):
+ lf.merge(self.load_lockfile(pre_lf_path))
- lf.update_tarball_resolutions(lambda p: self._contrib_tarball_url(p))
lf.write()
def _build_merged_workspace_config(self, ws, dep_paths):
diff --git a/build/plugins/lib/nots/package_manager/pnpm/utils.py b/build/plugins/lib/nots/package_manager/pnpm/utils.py
index 1fa4291b9d..9b6b9d80db 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/utils.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/utils.py
@@ -1,6 +1,10 @@
import os
-from .constants import PNPM_LOCKFILE_FILENAME, PNPM_WS_FILENAME
+from .constants import PNPM_PRE_LOCKFILE_FILENAME, PNPM_LOCKFILE_FILENAME, PNPM_WS_FILENAME
+
+
+def build_pre_lockfile_path(p):
+ return os.path.join(p, PNPM_PRE_LOCKFILE_FILENAME)
def build_lockfile_path(p):
diff --git a/build/plugins/lib/nots/package_manager/pnpm/workspace.py b/build/plugins/lib/nots/package_manager/pnpm/workspace.py
index e596e20a18..5639ae1c20 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/workspace.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/workspace.py
@@ -20,7 +20,8 @@ class PnpmWorkspace(object):
def read(self):
with open(self.path) as f:
- self.packages = set(yaml.load(f, Loader=yaml.CSafeLoader).get("packages", []))
+ parsed = yaml.load(f, Loader=yaml.CSafeLoader) or {}
+ self.packages = set(parsed.get("packages", []))
def write(self, path=None):
if not path:
diff --git a/build/plugins/lib/nots/typescript/tests/test_ts_config.py b/build/plugins/lib/nots/typescript/tests/test_ts_config.py
index cf67ca5ff9..43ae2d6738 100644
--- a/build/plugins/lib/nots/typescript/tests/test_ts_config.py
+++ b/build/plugins/lib/nots/typescript/tests/test_ts_config.py
@@ -48,11 +48,7 @@ def test_ts_config_validate_empty():
def test_ts_config_declaration_with_dir():
cfg = TsConfig(path="/tsconfig.json")
cfg.data = {
- "compilerOptions": {
- "rootDir": "./src",
- "declaration": True,
- "declarationDir": "some/dir"
- },
+ "compilerOptions": {"rootDir": "./src", "declaration": True, "declarationDir": "some/dir"},
}
cfg.validate(use_outdir=False)
@@ -61,10 +57,7 @@ def test_ts_config_declaration_with_dir():
def test_ts_config_declaration_without_dir():
cfg = TsConfig(path="/tsconfig.json")
cfg.data = {
- "compilerOptions": {
- "rootDir": "./src",
- "declaration": True
- },
+ "compilerOptions": {"rootDir": "./src", "declaration": True},
}
# When outDir should not be used we got the error
@@ -79,11 +72,7 @@ def test_ts_config_declaration_without_dir():
def test_ts_config_declaration_with_outdir():
cfg = TsConfig(path="/tsconfig.json")
cfg.data = {
- "compilerOptions": {
- "rootDir": "./src",
- "outDir": "some/dir",
- "declaration": True
- },
+ "compilerOptions": {"rootDir": "./src", "outDir": "some/dir", "declaration": True},
}
# When we allow outDir it will be enought to set it
diff --git a/build/plugins/nots.py b/build/plugins/nots.py
index 6e99f5c10b..cc4561bb0b 100644
--- a/build/plugins/nots.py
+++ b/build/plugins/nots.py
@@ -422,10 +422,6 @@ def _add_test(unit, test_type, test_files, deps=None, test_record=None, test_cwd
if test_record:
full_test_record.update(test_record)
- for k, v in full_test_record.items():
- if not isinstance(v, str):
- logger.warn(k, "expected 'str', got:", type(v))
-
data = ytest.dump_test(unit, full_test_record)
if data:
unit.set_property(["DART_DATA", data])
@@ -480,6 +476,23 @@ def _select_matching_version(erm_json, resource_name, range_str, dep_is_required
@_with_report_configure_error
+def on_prepare_deps_configure(unit):
+ pm = _create_pm(unit)
+ pj = pm.load_package_json_from_dir(pm.sources_path)
+ has_deps = pj.has_dependencies()
+ ins, outs = pm.calc_prepare_deps_inouts(unit.get("_TARBALLS_STORE"), has_deps)
+
+ if pj.has_dependencies():
+ unit.onpeerdir(pm.get_local_peers_from_package_json())
+ __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
+ __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
+
+ else:
+ __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", [], sorted(outs)))
+ unit.set(["_PREPARE_DEPS_CMD", "$_PREPARE_NO_DEPS_CMD"])
+
+
+@_with_report_configure_error
def on_node_modules_configure(unit):
pm = _create_pm(unit)
pj = pm.load_package_json_from_dir(pm.sources_path)
@@ -487,21 +500,11 @@ def on_node_modules_configure(unit):
if pj.has_dependencies():
unit.onpeerdir(pm.get_local_peers_from_package_json())
local_cli = unit.get("TS_LOCAL_CLI") == "yes"
- errors, ins, outs = pm.calc_node_modules_inouts(local_cli)
-
- if errors:
- ymake.report_configure_error(
- "There are some issues with lockfiles.\n"
- + "Please contact support (https://nda.ya.ru/t/sNoSFsO76ygSXL),\n"
- + "providing following details:\n"
- + "\n---\n".join([str(err) for err in errors])
- )
- else:
- unit.on_set_node_modules_ins_outs(["IN"] + sorted(ins) + ["OUT"] + sorted(outs))
+ ins, outs = pm.calc_node_modules_inouts(local_cli)
- __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
- if not unit.get("TS_TEST_FOR"):
- __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
+ __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
+ if not unit.get("TS_TEST_FOR"):
+ __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
if pj.get_use_prebuilder():
lf = pm.load_lockfile_from_dir(pm.sources_path)
@@ -526,10 +529,6 @@ def on_node_modules_configure(unit):
]
)
- else:
- # default "noop" command
- unit.set(["_NODE_MODULES_CMD", "$TOUCH_UNIT"])
-
@_with_report_configure_error
def on_ts_test_for_configure(unit, test_runner, default_config, node_modules_filename):