feat(npm): build standalone npm module

f944a35c196f6f7b7d93b7d2e9716fcd57f85d9f

11 files changed, 399 insertions, 22 deletions

diff --git a/build/plugins/lib/nots/package_manager/__init__.py b/build/plugins/lib/nots/package_manager/__init__.py
index 11387ec27a..3e1de532e9 100644
--- a/build/plugins/lib/nots/package_manager/__init__.py
+++ b/build/plugins/lib/nots/package_manager/__init__.py
@@ -1,10 +1,35 @@
-from .base import bundle_node_modules, constants, extract_node_modules, PackageJson, utils, PackageManagerCommandError
+import typing
+
+from .base import (
+    bundle_node_modules,
+    constants,
+    extract_node_modules,
+    PackageJson,
+    utils,
+    PackageManagerCommandError,
+    BasePackageManager,
+    BaseLockfile,
+)
 from .base.package_json import PackageJsonWorkspaceError
 from .pnpm import PnpmPackageManager
+from .npm import NpmPackageManager
 
 manager = PnpmPackageManager
 
+
+def get_package_manager_type(key: typing.Literal["pnpm", "npm"]) -> typing.Type[BasePackageManager]:
+    if key == "pnpm":
+        return PnpmPackageManager
+    if key == "npm":
+        return NpmPackageManager
+    raise ValueError(f"Invalid package manager key: {key}")
+
+
 __all__ = [
+    "BaseLockfile",
+    "BasePackageManager",
+    "PnpmPackageManager",
+    "NpmPackageManager",
     "PackageJson",
     "PackageJsonWorkspaceError",
     "PackageManagerCommandError",
diff --git a/build/plugins/lib/nots/package_manager/base/package_manager.py b/build/plugins/lib/nots/package_manager/base/package_manager.py
index 1c7cedb662..06a61ffebf 100644
--- a/build/plugins/lib/nots/package_manager/base/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/base/package_manager.py
@@ -76,15 +76,25 @@ class BasePackageManager(object):
         pass
 
     @abstractmethod
-    def create_node_modules(self):
+    def create_node_modules(self, yatool_prebuilder_path=None, local_cli=False, bundle=True):
         pass
 
     @abstractmethod
-    def calc_node_modules_inouts(self):
+    def extract_packages_meta_from_lockfiles(self, lf_paths):
         pass
 
     @abstractmethod
-    def extract_packages_meta_from_lockfiles(self, lf_paths):
+    def calc_prepare_deps_inouts_and_resources(
+        self, store_path: str, has_deps: bool
+    ) -> tuple[list[str], list[str], list[str]]:
+        pass
+
+    @abstractmethod
+    def calc_node_modules_inouts(self, local_cli=False) -> tuple[list[str], list[str]]:
+        pass
+
+    @abstractmethod
+    def build_workspace(self, tarballs_store: str):
         pass
 
     def get_local_peers_from_package_json(self):
@@ -104,7 +114,7 @@ class BasePackageManager(object):
 
         return [p[prefix_len:] for p in pj.get_workspace_map(ignore_self=True).keys()]
 
-    def _exec_command(self, args, include_defaults=True, script_path=None):
+    def _exec_command(self, args, include_defaults=True, script_path=None, env=None):
         if not self.nodejs_bin_path:
             raise PackageManagerError("Unable to execute command: nodejs_bin_path is not configured")
 
@@ -114,11 +124,7 @@ class BasePackageManager(object):
             + (self._get_default_options() if include_defaults else [])
         )
         p = subprocess.Popen(
-            cmd,
-            cwd=self.build_path,
-            stdin=None,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
+            cmd, cwd=self.build_path, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env
         )
         stdout, stderr = p.communicate()
 
diff --git a/build/plugins/lib/nots/package_manager/npm/__init__.py b/build/plugins/lib/nots/package_manager/npm/__init__.py
new file mode 100644
index 0000000000..96e87ea8ce
--- /dev/null
+++ b/build/plugins/lib/nots/package_manager/npm/__init__.py
@@ -0,0 +1,6 @@
+from .npm_package_manager import NpmPackageManager
+
+
+__all__ = [
+    "NpmPackageManager",
+]
diff --git a/build/plugins/lib/nots/package_manager/npm/npm_constants.py b/build/plugins/lib/nots/package_manager/npm/npm_constants.py
new file mode 100644
index 0000000000..f61145705e
--- /dev/null
+++ b/build/plugins/lib/nots/package_manager/npm/npm_constants.py
@@ -0,0 +1,6 @@
+NPM_LOCKFILE_FILENAME = "package-lock.json"
+
+# This is a name of intermediate file that is used in TS_PREPARE_DEPS.
+# This file has a structure same to package-lock.json, but all tarballs
+# a set relative to the build root.
+NPM_PRE_LOCKFILE_FILENAME = "pre.package-lock.json"
diff --git a/build/plugins/lib/nots/package_manager/npm/npm_lockfile.py b/build/plugins/lib/nots/package_manager/npm/npm_lockfile.py
new file mode 100644
index 0000000000..f0ea05d2b7
--- /dev/null
+++ b/build/plugins/lib/nots/package_manager/npm/npm_lockfile.py
@@ -0,0 +1,133 @@
+import base64
+import json
+import os
+import io
+
+from six.moves.urllib import parse as urlparse
+from six import iteritems
+
+from ..base import BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError
+
+LOCKFILE_VERSION_FIELD = "lockfileVersion"
+
+
+class NpmLockfile(BaseLockfile):
+    def read(self):
+        with io.open(self.path, "rb") as f:
+            self.data = json.load(f) or {LOCKFILE_VERSION_FIELD: 3}
+
+            lockfile_version = self.data.get(LOCKFILE_VERSION_FIELD, "<no-version>")
+            if lockfile_version != 3:
+                raise Exception(
+                    f'Error of project configuration: {self.path} has lockfileVersion: {lockfile_version}. '
+                    + f'This version is not supported. Please, delete {os.path.basename(self.path)} and regenerate it using "ya tool nots --clean install --lockfile-only --npm"'
+                )
+
+    def write(self, path=None):
+        """
+        :param path: path to store lockfile, defaults to original path
+        :type path: str
+        """
+        if path is None:
+            path = self.path
+
+        with open(path, "w") as f:
+            json.dump(self.data, f, indent=2)
+
+    def get_packages_meta(self):
+        """
+        Extracts packages meta from lockfile.
+        :rtype: list of LockfilePackageMeta
+        """
+        packages = self.data.get("packages", {})
+
+        for key, meta in packages.items():
+            if not key:
+                continue
+            yield _parse_package_meta(key, meta)
+
+    def update_tarball_resolutions(self, fn):
+        """
+        :param fn: maps `LockfilePackageMeta` instance to new `resolution.tarball` value
+        :type fn: lambda
+        """
+        packages = self.data.get("packages", {})
+
+        for key, meta in iteritems(packages):
+            if not key:
+                continue
+            meta["resolved"] = fn(_parse_package_meta(key, meta, allow_file_protocol=True))
+            packages[key] = meta
+
+    def get_requires_build_packages(self):
+        raise NotImplementedError()
+
+    def validate_has_addons_flags(self):
+        raise NotImplementedError()
+
+
+def _parse_package_meta(key, meta, allow_file_protocol=False):
+    """
+    :param key: uniq package key from lockfile
+    :type key: string
+    :param meta: package meta dict from lockfile
+    :type meta: dict
+    :rtype: LockfilePackageMetaInvalidError
+    """
+    try:
+        tarball_url = _parse_tarball_url(meta["resolved"], allow_file_protocol)
+        sky_id = _parse_sky_id_from_tarball_url(meta["resolved"])
+        integrity_algorithm, integrity = _parse_package_integrity(meta["integrity"])
+    except KeyError as e:
+        raise TypeError("Invalid package meta for key '{}', missing '{}' key".format(key, e))
+    except LockfilePackageMetaInvalidError as e:
+        raise TypeError("Invalid package meta for key '{}', parse error: '{}'".format(key, e))
+
+    return LockfilePackageMeta(key, tarball_url, sky_id, integrity, integrity_algorithm)
+
+
+def _parse_tarball_url(tarball_url, allow_file_protocol):
+    if tarball_url.startswith("file:") and not allow_file_protocol:
+        raise LockfilePackageMetaInvalidError("tarball cannot point to a file, got {}".format(tarball_url))
+    return tarball_url.split("?")[0]
+
+
+def _parse_sky_id_from_tarball_url(tarball_url):
+    """
+    :param tarball_url: tarball url
+    :type tarball_url: string
+    :rtype: string
+    """
+    if tarball_url.startswith("file:"):
+        return ""
+
+    rbtorrent_param = urlparse.parse_qs(urlparse.urlparse(tarball_url).query).get("rbtorrent")
+
+    if rbtorrent_param is None:
+        return ""
+
+    return "rbtorrent:{}".format(rbtorrent_param[0])
+
+
+def _parse_package_integrity(integrity):
+    """
+    Returns tuple of algorithm and hash (hex).
+    :param integrity: package integrity in format "{algo}-{base64_of_hash}"
+    :type integrity: string
+    :rtype: (str, str)
+    """
+    algo, hash_b64 = integrity.split("-", 1)
+
+    if algo not in ("sha1", "sha512"):
+        raise LockfilePackageMetaInvalidError(
+            f"Invalid package integrity algorithm, expected one of ('sha1', 'sha512'), got '{algo}'"
+        )
+
+    try:
+        base64.b64decode(hash_b64)
+    except TypeError as e:
+        raise LockfilePackageMetaInvalidError(
+            "Invalid package integrity encoding, integrity: {}, error: {}".format(integrity, e)
+        )
+
+    return (algo, hash_b64)
diff --git a/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py b/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py
new file mode 100644
index 0000000000..10c477c4e8
--- /dev/null
+++ b/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py
@@ -0,0 +1,134 @@
+import os
+
+from ..base import BasePackageManager, PackageManagerError
+from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME
+from ..base.node_modules_bundler import bundle_node_modules
+from ..base.utils import b_rooted, build_nm_bundle_path, build_pj_path, s_rooted
+
+from .npm_lockfile import NpmLockfile
+from .npm_utils import build_lockfile_path, build_pre_lockfile_path
+
+
+class NpmPackageManager(BasePackageManager):
+    @classmethod
+    def load_lockfile(cls, path):
+        """
+        :param path: path to lockfile
+        :type path: str
+        :rtype: NpmLockfile
+        """
+        return NpmLockfile.load(path)
+
+    @classmethod
+    def load_lockfile_from_dir(cls, dir_path):
+        """
+        :param dir_path: path to directory with lockfile
+        :type dir_path: str
+        :rtype: NpmLockfile
+        """
+        return cls.load_lockfile(build_lockfile_path(dir_path))
+
+    def extract_packages_meta_from_lockfiles(self, lf_paths):
+        """
+        :type lf_paths: iterable of BaseLockfile
+        :rtype: iterable of LockfilePackageMeta
+        """
+        tarballs = set()
+        errors = []
+
+        for lf_path in lf_paths:
+            try:
+                for pkg in self.load_lockfile(lf_path).get_packages_meta():
+                    if pkg.tarball_path not in tarballs:
+                        tarballs.add(pkg.tarball_path)
+                        yield pkg
+            except Exception as e:
+                errors.append("{}: {}".format(lf_path, e))
+
+        if errors:
+            raise PackageManagerError("Unable to process some lockfiles:\n{}".format("\n".join(errors)))
+
+    def calc_prepare_deps_inouts(self, store_path: str, has_deps: bool) -> tuple[list[str], list[str]]:
+        raise NotImplementedError("NPM does not support contrib/typescript flow.")
+
+    def calc_prepare_deps_inouts_and_resources(
+        self, store_path: str, has_deps: bool
+    ) -> tuple[list[str], list[str], list[str]]:
+        ins = [
+            s_rooted(build_pj_path(self.module_path)),
+            s_rooted(build_lockfile_path(self.module_path)),
+        ]
+        outs = [
+            b_rooted(build_pre_lockfile_path(self.module_path)),
+        ]
+        resources = []
+
+        if has_deps:
+            for dep_path in self.get_local_peers_from_package_json():
+                ins.append(b_rooted(build_pre_lockfile_path(dep_path)))
+
+            for pkg in self.extract_packages_meta_from_lockfiles([build_lockfile_path(self.sources_path)]):
+                resources.append(pkg.to_uri())
+                outs.append(b_rooted(self._tarballs_store_path(pkg, store_path)))
+
+        return ins, outs, resources
+
+    def calc_node_modules_inouts(self, local_cli=False) -> tuple[list[str], list[str]]:
+        """
+        Returns input and output paths for command that creates `node_modules` bundle.
+        It relies on .PEERDIRSELF=TS_PREPARE_DEPS
+        Inputs:
+            - source package.json
+            - merged pre-lockfiles and workspace configs of TS_PREPARE_DEPS
+        Outputs:
+            - created node_modules bundle
+        """
+        ins = [s_rooted(build_pj_path(self.module_path))]
+        outs = []
+
+        pj = self.load_package_json_from_dir(self.sources_path)
+        if pj.has_dependencies():
+            ins.append(b_rooted(build_pre_lockfile_path(self.module_path)))
+            if not local_cli:
+                outs.append(b_rooted(build_nm_bundle_path(self.module_path)))
+            for dep_path in self.get_local_peers_from_package_json():
+                ins.append(b_rooted(build_pj_path(dep_path)))
+
+        return ins, outs
+
+    def build_workspace(self, tarballs_store: str):
+        self._build_pre_lockfile(tarballs_store)
+
+    def _build_pre_lockfile(self, tarballs_store: str):
+        lf = self.load_lockfile_from_dir(self.sources_path)
+        # Change to the output path for correct path calcs on merging.
+        lf.path = build_pre_lockfile_path(self.build_path)
+        lf.update_tarball_resolutions(lambda p: self._tarballs_store_path(p, tarballs_store))
+
+        lf.write()
+
+    def create_node_modules(self, yatool_prebuilder_path=None, local_cli=False, bundle=True):
+        """
+        Creates node_modules directory according to the lockfile.
+        """
+        self._prepare_workspace()
+
+        install_cmd = ["clean-install", "--ignore-scripts", "--audit=false"]
+
+        env = os.environ.copy()
+        env.update({"NPM_CONFIG_CACHE": os.path.join(self.build_path, ".npm-cache")})
+
+        self._exec_command(install_cmd, env=env)
+
+        if not local_cli and bundle:
+            bundle_node_modules(
+                build_root=self.build_root,
+                node_modules_path=self._nm_path(),
+                peers=[],
+                bundle_path=os.path.join(self.build_path, NODE_MODULES_WORKSPACE_BUNDLE_FILENAME),
+            )
+
+    def _prepare_workspace(self):
+        lf = self.load_lockfile(build_pre_lockfile_path(self.build_path))
+        lf.update_tarball_resolutions(lambda p: "file:" + os.path.join(self.build_root, p.tarball_url))
+        lf.write(build_lockfile_path(self.build_path))
diff --git a/build/plugins/lib/nots/package_manager/npm/npm_utils.py b/build/plugins/lib/nots/package_manager/npm/npm_utils.py
new file mode 100644
index 0000000000..b58686b3b2
--- /dev/null
+++ b/build/plugins/lib/nots/package_manager/npm/npm_utils.py
@@ -0,0 +1,11 @@
+import os
+
+from .npm_constants import NPM_LOCKFILE_FILENAME, NPM_PRE_LOCKFILE_FILENAME
+
+
+def build_pre_lockfile_path(p):
+    return os.path.join(p, NPM_PRE_LOCKFILE_FILENAME)
+
+
+def build_lockfile_path(p):
+    return os.path.join(p, NPM_LOCKFILE_FILENAME)
diff --git a/build/plugins/lib/nots/package_manager/npm/ya.make b/build/plugins/lib/nots/package_manager/npm/ya.make
new file mode 100644
index 0000000000..1ab91b8d69
--- /dev/null
+++ b/build/plugins/lib/nots/package_manager/npm/ya.make
@@ -0,0 +1,19 @@
+SUBSCRIBER(g:frontend_build_platform)
+
+PY3_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+    __init__.py
+    npm_constants.py
+    npm_lockfile.py
+    npm_package_manager.py
+    npm_utils.py
+)
+
+PEERDIR(
+    build/plugins/lib/nots/package_manager/base
+)
+
+END()
diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
index 7022c2d9ed..ea82b3637f 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
@@ -142,7 +142,7 @@ class PnpmPackageManager(BasePackageManager):
         It relies on .PEERDIRSELF=TS_PREPARE_DEPS
         Inputs:
             - source package.json
-            - merged lockfiles and workspace configs of TS_PREPARE_DEPS
+            - merged pre-lockfiles and workspace configs of TS_PREPARE_DEPS
         Outputs:
             - created node_modules bundle
         """
@@ -187,7 +187,7 @@ class PnpmPackageManager(BasePackageManager):
 
         return PnpmWorkspace.load(build_ws_config_path(self.build_path))
 
-    def build_workspace(self, tarballs_store):
+    def build_workspace(self, tarballs_store: str):
         """
         :rtype: PnpmWorkspace
         """
diff --git a/build/plugins/lib/nots/package_manager/ya.make b/build/plugins/lib/nots/package_manager/ya.make
index a77af04676..08a791eb7e 100644
--- a/build/plugins/lib/nots/package_manager/ya.make
+++ b/build/plugins/lib/nots/package_manager/ya.make
@@ -12,6 +12,7 @@ PY_SRCS(
 PEERDIR(
     build/plugins/lib/nots/package_manager/base
     build/plugins/lib/nots/package_manager/pnpm
+    build/plugins/lib/nots/package_manager/npm
 )
 
 END()
@@ -19,4 +20,5 @@ END()
 RECURSE(
     base
     pnpm
+    npm
 )
diff --git a/build/plugins/nots.py b/build/plugins/nots.py
index 4a2f8474da..b99de011d4 100644
--- a/build/plugins/nots.py
+++ b/build/plugins/nots.py
@@ -4,7 +4,7 @@ import os
 import ymake
 import ytest
 
-from _common import resolve_common_const, get_norm_unit_path, rootrel_arc_src, to_yesno
+from _common import resolve_common_const, get_norm_unit_path, rootrel_arc_src, strip_roots, to_yesno
 
 
 # 1 is 60 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL)
@@ -118,16 +118,28 @@ def _build_cmd_input_paths(paths, hide=False, disable_include_processor=False):
     return _build_directives("input", [hide_part, disable_ip_part], paths)
 
 
+def _get_pm_type(unit) -> str:
+    resolved = unit.get("PM_TYPE")
+    if not resolved:
+        raise Exception("PM_TYPE is not set yet. Macro _SET_PACKAGE_MANAGER() should be called before.")
+
+    return resolved
+
+
+def _get_source_path(unit):
+    sources_path = unit.get("TS_TEST_FOR_DIR") if unit.get("TS_TEST_FOR") else unit.path()
+    return sources_path
+
+
 def _create_pm(unit):
-    from lib.nots.package_manager import manager
+    from lib.nots.package_manager import get_package_manager_type
+
+    sources_path = _get_source_path(unit)
+    module_path = unit.get("TS_TEST_FOR_PATH") if unit.get("TS_TEST_FOR") else unit.get("MODDIR")
 
-    sources_path = unit.path()
-    module_path = unit.get("MODDIR")
-    if unit.get("TS_TEST_FOR"):
-        sources_path = unit.get("TS_TEST_FOR_DIR")
-        module_path = unit.get("TS_TEST_FOR_PATH")
+    PackageManager = get_package_manager_type(_get_pm_type(unit))
 
-    return manager(
+    return PackageManager(
         sources_path=unit.resolve(sources_path),
         build_root="$B",
         build_path=unit.path().replace("$S", "$B", 1),
@@ -148,6 +160,25 @@ def _create_erm_json(unit):
 
 
 @_with_report_configure_error
+def on_set_package_manager(unit):
+    pm_type = "pnpm"  # projects without any lockfile are processed by pnpm
+
+    source_path = _get_source_path(unit)
+
+    for pm_key, lockfile_name in [("pnpm", "pnpm-lock.yaml"), ("npm", "package-lock.json")]:
+        lf_path = os.path.join(source_path, lockfile_name)
+        lf_path_resolved = unit.resolve_arc_path(strip_roots(lf_path))
+
+        if lf_path_resolved:
+            pm_type = pm_key
+            break
+
+    unit.on_peerdir_ts_resource(pm_type)
+    unit.set(["PM_TYPE", pm_type])
+    unit.set(["PM_SCRIPT", f"${pm_type.upper()}_SCRIPT"])
+
+
+@_with_report_configure_error
 def on_set_append_with_directive(unit, var_name, dir, *values):
     wrapped = ['${{{dir}:"{v}"}}'.format(dir=dir, v=v) for v in values]
     __set_append(unit, var_name, " ".join(wrapped))
@@ -157,6 +188,9 @@ def on_set_append_with_directive(unit, var_name, dir, *values):
 def on_from_npm_lockfiles(unit, *args):
     from lib.nots.package_manager.base import PackageManagerError
 
+    # This is contrib with pnpm-lock.yaml files only
+    # Force set to pnpm
+    unit.set(["PM_TYPE", "pnpm"])
     pm = _create_pm(unit)
     lf_paths = []
 
@@ -192,8 +226,9 @@ def _check_nodejs_version(unit, major):
 
 @_with_report_configure_error
 def on_peerdir_ts_resource(unit, *resources):
-    pm = _create_pm(unit)
-    pj = pm.load_package_json_from_dir(pm.sources_path)
+    from lib.nots.package_manager import BasePackageManager
+
+    pj = BasePackageManager.load_package_json_from_dir(unit.resolve(_get_source_path(unit)))
     erm_json = _create_erm_json(unit)
     dirs = []