feat(conf): configure used tarballs by own pnpm-lock.yaml

b61ff371d6ad2f8c646f2a04e4a7d0fa178a6541

11 files changed, 104 insertions, 111 deletions

diff --git a/build/plugins/lib/nots/package_manager/base/lockfile.py b/build/plugins/lib/nots/package_manager/base/lockfile.py
index f13168b320..b6b9952602 100644
--- a/build/plugins/lib/nots/package_manager/base/lockfile.py
+++ b/build/plugins/lib/nots/package_manager/base/lockfile.py
@@ -9,16 +9,17 @@ class LockfilePackageMeta(object):
     Basic struct representing package meta from lockfile.
     """
 
-    __slots__ = ("tarball_url", "sky_id", "integrity", "integrity_algorithm", "tarball_path")
+    __slots__ = ("key", "tarball_url", "sky_id", "integrity", "integrity_algorithm", "tarball_path")
 
     @staticmethod
     def from_str(s):
         return LockfilePackageMeta(*s.strip().split(" "))
 
-    def __init__(self, tarball_url, sky_id, integrity, integrity_algorithm):
+    def __init__(self, key, tarball_url, sky_id, integrity, integrity_algorithm):
         # http://npm.yandex-team.ru/@scope%2fname/-/name-0.0.1.tgz
         parts = tarball_url.split("/")
 
+        self.key = key
         self.tarball_url = tarball_url
         self.sky_id = sky_id
         self.integrity = integrity
diff --git a/build/plugins/lib/nots/package_manager/base/package_json.py b/build/plugins/lib/nots/package_manager/base/package_json.py
index cc498b33f7..2438511800 100644
--- a/build/plugins/lib/nots/package_manager/base/package_json.py
+++ b/build/plugins/lib/nots/package_manager/base/package_json.py
@@ -130,10 +130,12 @@ class PackageJson(object):
 
         return None
 
+    # TODO: FBP-1254
+    # def get_workspace_dep_spec_paths(self) -> list[tuple[str, str]]:
     def get_workspace_dep_spec_paths(self):
         """
         Returns names and paths from specifiers of the defined workspace dependencies.
-        :rtype: list of (str, str)
+        :rtype: list[tuple[str, str]]
         """
         spec_paths = []
         schema = self.WORKSPACE_SCHEMA
diff --git a/build/plugins/lib/nots/package_manager/base/package_manager.py b/build/plugins/lib/nots/package_manager/base/package_manager.py
index 6b9faa56e8..1c7cedb662 100644
--- a/build/plugins/lib/nots/package_manager/base/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/base/package_manager.py
@@ -133,8 +133,8 @@ class BasePackageManager(object):
     def _contrib_tarball_path(self, pkg):
         return os.path.join(self.contribs_path, pkg.tarball_path)
 
-    def _contrib_tarball_url(self, pkg):
-        return "file:" + self._contrib_tarball_path(pkg)
+    def _tarballs_store_path(self, pkg, store_path):
+        return os.path.join(self.module_path, store_path, pkg.tarball_path)
 
     def _get_default_options(self):
         return ["--registry", NPM_REGISTRY_URL]
diff --git a/build/plugins/lib/nots/package_manager/pnpm/__init__.py b/build/plugins/lib/nots/package_manager/pnpm/__init__.py
index af6de8e62a..0f50359c47 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/__init__.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/__init__.py
@@ -1,11 +1,12 @@
 from . import constants
 from .lockfile import PnpmLockfile
 from .package_manager import PnpmPackageManager
-from .utils import build_ws_config_path
+from .utils import build_ws_config_path, build_lockfile_path
 from .workspace import PnpmWorkspace
 
 
 __all__ = [
+    "build_lockfile_path",
     "build_ws_config_path",
     "constants",
     "PnpmLockfile",
diff --git a/build/plugins/lib/nots/package_manager/pnpm/constants.py b/build/plugins/lib/nots/package_manager/pnpm/constants.py
index e84a78c55e..10ca9e9272 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/constants.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/constants.py
@@ -1,2 +1,7 @@
 PNPM_WS_FILENAME = "pnpm-workspace.yaml"
 PNPM_LOCKFILE_FILENAME = "pnpm-lock.yaml"
+
+# This is a name of intermediate file that is used in TS_PREPARE_DEPS.
+# This file has a structure same to pnpm-lock.yaml, but all tarballs
+# a set relative to the build root.
+PNPM_PRE_LOCKFILE_FILENAME = "pre.pnpm-lock.yaml"
diff --git a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
index eca1e4015b..46558861bd 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
@@ -10,21 +10,22 @@ from six import iteritems
 
 from ..base import PackageJson, BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError
 
+LOCKFILE_VERSION = "lockfileVersion"
+
 
 class PnpmLockfile(BaseLockfile):
     IMPORTER_KEYS = PackageJson.DEP_KEYS + ("specifiers",)
 
     def read(self):
         with io.open(self.path, "rb") as f:
-            self.data = yaml.load(f, Loader=yaml.CSafeLoader)
+            self.data = yaml.load(f, Loader=yaml.CSafeLoader) or {LOCKFILE_VERSION: "6.0"}
 
-            lockfileVersion = "lockfileVersion"
-            version_in_data = lockfileVersion in self.data
+            version_in_data = LOCKFILE_VERSION in self.data
             r = re.compile('^[56]\\.\\d$')
-            if not version_in_data or not r.match(str(self.data[lockfileVersion])):
+            if not version_in_data or not r.match(str(self.data[LOCKFILE_VERSION])):
                 raise Exception(
                     'Error of project configuration: {} has lockfileVersion: {}. '.format(
-                        self.path, self.data[lockfileVersion] if version_in_data else "<no-version>"
+                        self.path, self.data[LOCKFILE_VERSION] if version_in_data else "<no-version>"
                     )
                     + 'This version is not supported. Please, delete pnpm-lock.yaml and regenerate it using "ya tool nots --clean update-lockfile"'
                 )
@@ -129,7 +130,7 @@ def _parse_package_meta(key, meta, allow_file_protocol=False):
     except LockfilePackageMetaInvalidError as e:
         raise TypeError("Invalid package meta for key {}, parse error: {}".format(key, e))
 
-    return LockfilePackageMeta(tarball_url, sky_id, integrity, integrity_algorithm)
+    return LockfilePackageMeta(key, tarball_url, sky_id, integrity, integrity_algorithm)
 
 
 def _parse_tarball_url(tarball_url, allow_file_protocol):
diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
index 3568dbff16..00151f4f51 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
@@ -1,10 +1,8 @@
 import os
 import shutil
 
-from six import iteritems
-
 from .lockfile import PnpmLockfile
-from .utils import build_lockfile_path, build_ws_config_path
+from .utils import build_pre_lockfile_path, build_lockfile_path, build_ws_config_path
 from .workspace import PnpmWorkspace
 from ..base import BasePackageManager, PackageManagerError
 from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME
@@ -82,69 +80,54 @@ class PnpmPackageManager(BasePackageManager):
                 bundle_path=os.path.join(self.build_path, NODE_MODULES_WORKSPACE_BUNDLE_FILENAME),
             )
 
+    # TODO: FBP-1254
+    # def calc_prepare_deps_inouts(self, store_path: str, has_deps: bool) -> (list[str], list[str]):
+    def calc_prepare_deps_inouts(self, store_path, has_deps):
+        ins = [
+            s_rooted(build_pj_path(self.module_path)),
+            s_rooted(build_lockfile_path(self.module_path)),
+        ]
+        outs = [
+            b_rooted(build_ws_config_path(self.module_path)),
+            b_rooted(build_pre_lockfile_path(self.module_path)),
+        ]
+
+        if has_deps:
+            for dep_path in self.get_local_peers_from_package_json():
+                ins.append(b_rooted(build_ws_config_path(dep_path)))
+                ins.append(b_rooted(build_pre_lockfile_path(dep_path)))
+
+            for pkg in self.extract_packages_meta_from_lockfiles([build_lockfile_path(self.sources_path)]):
+                ins.append(b_rooted(self._contrib_tarball_path(pkg)))
+                outs.append(b_rooted(self._tarballs_store_path(pkg, store_path)))
+
+        return ins, outs
+
+    # TODO: FBP-1254
+    # def calc_node_modules_inouts(self, local_cli=False) -> (list[str], list[str]):
     def calc_node_modules_inouts(self, local_cli=False):
         """
         Returns input and output paths for command that creates `node_modules` bundle.
-        Errors: errors caught while processing lockfiles
+        It relies on .PEERDIRSELF=TS_PREPARE_DEPS
         Inputs:
-            - source package.json and lockfile,
-            - built package.jsons of all deps,
-            - merged lockfiles and workspace configs of direct non-leave deps,
-            - tarballs.
+            - source package.json
+            - merged lockfiles and workspace configs of TS_PREPARE_DEPS
         Outputs:
-            - merged lockfile,
-            - generated workspace config,
-            - created node_modules bundle.
-        :rtype: (list of errors, list of str, list of str)
+            - created node_modules bundle
         """
-        ins = [
-            s_rooted(build_pj_path(self.module_path)),
-        ]
+        ins = [s_rooted(build_pj_path(self.module_path))]
         outs = []
 
         pj = self.load_package_json_from_dir(self.sources_path)
         if pj.has_dependencies():
-            ins.extend(
-                [
-                    s_rooted(build_lockfile_path(self.module_path)),
-                ]
-            )
-            outs.extend(
-                [
-                    b_rooted(build_lockfile_path(self.module_path)),
-                    b_rooted(build_ws_config_path(self.module_path)),
-                ]
-            )
+            ins.append(b_rooted(build_pre_lockfile_path(self.module_path)))
+            ins.append(b_rooted(build_ws_config_path(self.module_path)))
             if not local_cli:
-                outs.extend([b_rooted(build_nm_bundle_path(self.module_path))])
-
-        # Source lockfiles are used only to get tarballs info.
-        src_lf_paths = [build_lockfile_path(self.sources_path)]
-
-        for [dep_src_path, (_, depth)] in iteritems(pj.get_workspace_map(ignore_self=True)):
-            dep_mod_path = dep_src_path[len(self.sources_root) + 1 :]
-            # pnpm requires all package.jsons.
-            ins.append(b_rooted(build_pj_path(dep_mod_path)))
-
-            dep_lf_src_path = build_lockfile_path(dep_src_path)
-            if not os.path.isfile(dep_lf_src_path):
-                # It is ok for leaves.
-                continue
-            src_lf_paths.append(dep_lf_src_path)
+                outs.append(b_rooted(build_nm_bundle_path(self.module_path)))
+            for dep_path in self.get_local_peers_from_package_json():
+                ins.append(b_rooted(build_pj_path(dep_path)))
 
-            if depth == 1:
-                ins.append(b_rooted(build_ws_config_path(dep_mod_path)))
-                ins.append(b_rooted(build_lockfile_path(dep_mod_path)))
-
-        errors = []
-        try:
-            for pkg in self.extract_packages_meta_from_lockfiles(src_lf_paths):
-                ins.append(b_rooted(self._contrib_tarball_path(pkg)))
-        except Exception as e:
-            errors.append(e)
-            pass
-
-        return errors, ins, outs
+        return ins, outs
 
     def extract_packages_meta_from_lockfiles(self, lf_paths):
         """
@@ -167,6 +150,13 @@ class PnpmPackageManager(BasePackageManager):
             raise PackageManagerError("Unable to process some lockfiles:\n{}".format("\n".join(errors)))
 
     def _prepare_workspace(self):
+        lf = self.load_lockfile(build_pre_lockfile_path(self.build_path))
+        lf.update_tarball_resolutions(lambda p: "file:" + os.path.join(self.build_root, p.tarball_url))
+        lf.write(build_lockfile_path(self.build_path))
+
+        return PnpmWorkspace.load(build_ws_config_path(self.build_path))
+
+    def build_workspace(self, tarballs_store):
         """
         :rtype: PnpmWorkspace
         """
@@ -177,7 +167,7 @@ class PnpmPackageManager(BasePackageManager):
 
         dep_paths = ws.get_paths(ignore_self=True)
         self._build_merged_workspace_config(ws, dep_paths)
-        self._build_merged_lockfile(dep_paths)
+        self._build_merged_pre_lockfile(tarballs_store, dep_paths)
 
         return ws
 
@@ -195,21 +185,21 @@ class PnpmPackageManager(BasePackageManager):
 
         return pj
 
-    def _build_merged_lockfile(self, dep_paths):
+    def _build_merged_pre_lockfile(self, tarballs_store, dep_paths):
         """
         :type dep_paths: list of str
         :rtype: PnpmLockfile
         """
         lf = self.load_lockfile_from_dir(self.sources_path)
         # Change to the output path for correct path calcs on merging.
-        lf.path = build_lockfile_path(self.build_path)
+        lf.path = build_pre_lockfile_path(self.build_path)
+        lf.update_tarball_resolutions(lambda p: self._tarballs_store_path(p, tarballs_store))
 
         for dep_path in dep_paths:
-            lf_path = build_lockfile_path(dep_path)
-            if os.path.isfile(lf_path):
-                lf.merge(self.load_lockfile(lf_path))
+            pre_lf_path = build_pre_lockfile_path(dep_path)
+            if os.path.isfile(pre_lf_path):
+                lf.merge(self.load_lockfile(pre_lf_path))
 
-        lf.update_tarball_resolutions(lambda p: self._contrib_tarball_url(p))
         lf.write()
 
     def _build_merged_workspace_config(self, ws, dep_paths):
diff --git a/build/plugins/lib/nots/package_manager/pnpm/utils.py b/build/plugins/lib/nots/package_manager/pnpm/utils.py
index 1fa4291b9d..9b6b9d80db 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/utils.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/utils.py
@@ -1,6 +1,10 @@
 import os
 
-from .constants import PNPM_LOCKFILE_FILENAME, PNPM_WS_FILENAME
+from .constants import PNPM_PRE_LOCKFILE_FILENAME, PNPM_LOCKFILE_FILENAME, PNPM_WS_FILENAME
+
+
+def build_pre_lockfile_path(p):
+    return os.path.join(p, PNPM_PRE_LOCKFILE_FILENAME)
 
 
 def build_lockfile_path(p):
diff --git a/build/plugins/lib/nots/package_manager/pnpm/workspace.py b/build/plugins/lib/nots/package_manager/pnpm/workspace.py
index e596e20a18..5639ae1c20 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/workspace.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/workspace.py
@@ -20,7 +20,8 @@ class PnpmWorkspace(object):
 
     def read(self):
         with open(self.path) as f:
-            self.packages = set(yaml.load(f, Loader=yaml.CSafeLoader).get("packages", []))
+            parsed = yaml.load(f, Loader=yaml.CSafeLoader) or {}
+            self.packages = set(parsed.get("packages", []))
 
     def write(self, path=None):
         if not path:
diff --git a/build/plugins/lib/nots/typescript/tests/test_ts_config.py b/build/plugins/lib/nots/typescript/tests/test_ts_config.py
index cf67ca5ff9..43ae2d6738 100644
--- a/build/plugins/lib/nots/typescript/tests/test_ts_config.py
+++ b/build/plugins/lib/nots/typescript/tests/test_ts_config.py
@@ -48,11 +48,7 @@ def test_ts_config_validate_empty():
 def test_ts_config_declaration_with_dir():
     cfg = TsConfig(path="/tsconfig.json")
     cfg.data = {
-        "compilerOptions": {
-            "rootDir": "./src",
-            "declaration": True,
-            "declarationDir": "some/dir"
-        },
+        "compilerOptions": {"rootDir": "./src", "declaration": True, "declarationDir": "some/dir"},
     }
 
     cfg.validate(use_outdir=False)
@@ -61,10 +57,7 @@ def test_ts_config_declaration_with_dir():
 def test_ts_config_declaration_without_dir():
     cfg = TsConfig(path="/tsconfig.json")
     cfg.data = {
-        "compilerOptions": {
-            "rootDir": "./src",
-            "declaration": True
-        },
+        "compilerOptions": {"rootDir": "./src", "declaration": True},
     }
 
     # When outDir should not be used we got the error
@@ -79,11 +72,7 @@ def test_ts_config_declaration_without_dir():
 def test_ts_config_declaration_with_outdir():
     cfg = TsConfig(path="/tsconfig.json")
     cfg.data = {
-        "compilerOptions": {
-            "rootDir": "./src",
-            "outDir": "some/dir",
-            "declaration": True
-        },
+        "compilerOptions": {"rootDir": "./src", "outDir": "some/dir", "declaration": True},
     }
 
     # When we allow outDir it will be enought to set it
diff --git a/build/plugins/nots.py b/build/plugins/nots.py
index 6e99f5c10b..cc4561bb0b 100644
--- a/build/plugins/nots.py
+++ b/build/plugins/nots.py
@@ -422,10 +422,6 @@ def _add_test(unit, test_type, test_files, deps=None, test_record=None, test_cwd
     if test_record:
         full_test_record.update(test_record)
 
-    for k, v in full_test_record.items():
-        if not isinstance(v, str):
-            logger.warn(k, "expected 'str', got:", type(v))
-
     data = ytest.dump_test(unit, full_test_record)
     if data:
         unit.set_property(["DART_DATA", data])
@@ -480,6 +476,23 @@ def _select_matching_version(erm_json, resource_name, range_str, dep_is_required
 
 
 @_with_report_configure_error
+def on_prepare_deps_configure(unit):
+    pm = _create_pm(unit)
+    pj = pm.load_package_json_from_dir(pm.sources_path)
+    has_deps = pj.has_dependencies()
+    ins, outs = pm.calc_prepare_deps_inouts(unit.get("_TARBALLS_STORE"), has_deps)
+
+    if pj.has_dependencies():
+        unit.onpeerdir(pm.get_local_peers_from_package_json())
+        __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
+        __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
+
+    else:
+        __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", [], sorted(outs)))
+        unit.set(["_PREPARE_DEPS_CMD", "$_PREPARE_NO_DEPS_CMD"])
+
+
+@_with_report_configure_error
 def on_node_modules_configure(unit):
     pm = _create_pm(unit)
     pj = pm.load_package_json_from_dir(pm.sources_path)
@@ -487,21 +500,11 @@ def on_node_modules_configure(unit):
     if pj.has_dependencies():
         unit.onpeerdir(pm.get_local_peers_from_package_json())
         local_cli = unit.get("TS_LOCAL_CLI") == "yes"
-        errors, ins, outs = pm.calc_node_modules_inouts(local_cli)
-
-        if errors:
-            ymake.report_configure_error(
-                "There are some issues with lockfiles.\n"
-                + "Please contact support (https://nda.ya.ru/t/sNoSFsO76ygSXL),\n"
-                + "providing following details:\n"
-                + "\n---\n".join([str(err) for err in errors])
-            )
-        else:
-            unit.on_set_node_modules_ins_outs(["IN"] + sorted(ins) + ["OUT"] + sorted(outs))
+        ins, outs = pm.calc_node_modules_inouts(local_cli)
 
-            __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
-            if not unit.get("TS_TEST_FOR"):
-                __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
+        __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
+        if not unit.get("TS_TEST_FOR"):
+            __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
 
         if pj.get_use_prebuilder():
             lf = pm.load_lockfile_from_dir(pm.sources_path)
@@ -526,10 +529,6 @@ def on_node_modules_configure(unit):
                 ]
             )
 
-    else:
-        # default "noop" command
-        unit.set(["_NODE_MODULES_CMD", "$TOUCH_UNIT"])
-
 
 @_with_report_configure_error
 def on_ts_test_for_configure(unit, test_runner, default_config, node_modules_filename):