feat(FROM_NPM): load packages by tarball url

Раньше FROM_NPM строил урл для скачивания по имени пакета. Изменил это на использование поля tarball, потому что это требуется для интеграции с `@yatool/prebuilder`

7 files changed, 39 insertions, 47 deletions

diff --git a/build/conf/ts/node_modules.conf b/build/conf/ts/node_modules.conf
index e2b8bf4995..54d4b8d85e 100644
--- a/build/conf/ts/node_modules.conf
+++ b/build/conf/ts/node_modules.conf
@@ -86,8 +86,8 @@ macro FROM_NPM_LOCKFILES(LOCKFILES...) {
 }
 
 FROM_NPM_CWD=$ARCADIA_BUILD_ROOT/$NPM_CONTRIBS_PATH
-macro _FROM_NPM(NAME, VERSION, SKY_ID, INTEGRITY, INTEGRITY_ALGO, TARBALL_PATH) {
-    .CMD=${cwd:FROM_NPM_CWD} $YMAKE_PYTHON ${input:"build/scripts/fetch_from_npm.py"} ${input;hide:"build/scripts/fetch_from.py"} ${input;hide:"build/scripts/sky.py"} --name $NAME --version $VERSION --sky-id $SKY_ID --integrity $INTEGRITY --integrity-algorithm $INTEGRITY_ALGO --copy-to ${output;noauto:TARBALL_PATH} ${requirements;hide:"network:full"} ${kv;hide:"p NPM"} ${kv;hide:"pc magenta"}
+macro _FROM_NPM(TARBALL_URL, SKY_ID, INTEGRITY, INTEGRITY_ALGO, TARBALL_PATH) {
+    .CMD=${cwd:FROM_NPM_CWD} $YMAKE_PYTHON ${input:"build/scripts/fetch_from_npm.py"} ${input;hide:"build/scripts/fetch_from.py"} ${input;hide:"build/scripts/sky.py"} --tarball-url $TARBALL_URL --sky-id $SKY_ID --integrity $INTEGRITY --integrity-algorithm $INTEGRITY_ALGO --copy-to ${output;noauto:TARBALL_PATH} ${requirements;hide:"network:full"} ${kv;hide:"p NPM"} ${kv;hide:"pc magenta"}
     # we want output to be available for other modules without affecting NPM_CONTRIBS
     # we need to expose it (some details in https://st.yandex-team.ru/YMAKE-34)
     _EXPOSE($TARBALL_PATH)
diff --git a/build/plugins/lib/nots/package_manager/base/lockfile.py b/build/plugins/lib/nots/package_manager/base/lockfile.py
index 1d7cc6ad3e..10cfcc0fc6 100644
--- a/build/plugins/lib/nots/package_manager/base/lockfile.py
+++ b/build/plugins/lib/nots/package_manager/base/lockfile.py
@@ -9,22 +9,24 @@ class LockfilePackageMeta(object):
     Basic struct representing package meta from lockfile.
     """
 
-    __slots__ = ("name", "version", "sky_id", "integrity", "integrity_algorithm", "tarball_path")
+    __slots__ = ("tarball_url", "sky_id", "integrity", "integrity_algorithm", "tarball_path")
 
     @staticmethod
     def from_str(s):
         return LockfilePackageMeta(*s.strip().split(" "))
 
-    def __init__(self, name, version, sky_id, integrity, integrity_algorithm):
-        self.name = name
-        self.version = version
+    def __init__(self, tarball_url, sky_id, integrity, integrity_algorithm):
+        # http://npm.yandex-team.ru/@scope%2fname/-/name-0.0.1.tgz
+        parts = tarball_url.split("/")
+
+        self.tarball_url = tarball_url
         self.sky_id = sky_id
         self.integrity = integrity
         self.integrity_algorithm = integrity_algorithm
-        self.tarball_path = "{}-{}.tgz".format(name, version)
+        self.tarball_path = "/".join(parts[-3:])  # @scope%2fname/-/name-0.0.1.tgz
 
     def to_str(self):
-        return " ".join([self.name, self.version, self.sky_id, self.integrity, self.integrity_algorithm])
+        return " ".join([self.tarball_url, self.sky_id, self.integrity, self.integrity_algorithm])
 
 
 class LockfilePackageMetaInvalidError(RuntimeError):
@@ -61,7 +63,7 @@ class BaseLockfile(object):
         pass
 
     @abstractmethod
-    def get_packages_meta(self):
+    def get_packages_meta(self, no_files):
         pass
 
     @abstractmethod
diff --git a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
index 79c351b7fa..5e55a6f661 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
@@ -27,14 +27,14 @@ class PnpmLockfile(BaseLockfile):
         with open(path, "w") as f:
             yaml.dump(self.data, f, Dumper=yaml.CSafeDumper)
 
-    def get_packages_meta(self):
+    def get_packages_meta(self, no_files):
         """
         Extracts packages meta from lockfile.
         :rtype: list of LockfilePackageMeta
         """
         packages = self.data.get("packages", {})
 
-        return map(lambda x: _parse_package_meta(*x), iteritems(packages))
+        return map(lambda x: _parse_package_meta(x[0], x[1], no_files), iteritems(packages))
 
     def update_tarball_resolutions(self, fn):
         """
@@ -44,7 +44,7 @@ class PnpmLockfile(BaseLockfile):
         packages = self.data.get("packages", {})
 
         for key, meta in iteritems(packages):
-            meta["resolution"]["tarball"] = fn(_parse_package_meta(key, meta))
+            meta["resolution"]["tarball"] = fn(_parse_package_meta(key, meta, no_files=False))
             packages[key] = meta
 
     def get_importers(self):
@@ -89,7 +89,7 @@ class PnpmLockfile(BaseLockfile):
         self.data["packages"] = packages
 
 
-def _parse_package_meta(key, meta):
+def _parse_package_meta(key, meta, no_files):
     """
     :param key: uniq package key from lockfile
     :type key: string
@@ -98,7 +98,7 @@ def _parse_package_meta(key, meta):
     :rtype: LockfilePackageMetaInvalidError
     """
     try:
-        name, version = _parse_package_key(key)
+        tarball_url = _parse_tarball_url(meta["resolution"]["tarball"], no_files)
         sky_id = _parse_sky_id_from_tarball_url(meta["resolution"]["tarball"])
         integrity_algorithm, integrity = _parse_package_integrity(meta["resolution"]["integrity"])
     except KeyError as e:
@@ -106,26 +106,13 @@ def _parse_package_meta(key, meta):
     except LockfilePackageMetaInvalidError as e:
         raise TypeError("Invalid package meta for key {}, parse error: {}".format(key, e))
 
-    return LockfilePackageMeta(name, version, sky_id, integrity, integrity_algorithm)
+    return LockfilePackageMeta(tarball_url, sky_id, integrity, integrity_algorithm)
 
 
-def _parse_package_key(key):
-    """
-    Returns tuple of scoped package name and version.
-    :param key: package key in format "/({scope}/)?{package_name}/{package_version}(_{peer_dependencies})?"
-    :type key: string
-    :rtype: (str, str)
-    """
-    try:
-        tokens = key.split("/")[1:]
-        version = tokens.pop().split("_", 1)[0]
-
-        if len(tokens) < 1 or len(tokens) > 2:
-            raise TypeError()
-    except (IndexError, TypeError):
-        raise LockfilePackageMetaInvalidError("Invalid package key")
-
-    return ("/".join(tokens), version)
+def _parse_tarball_url(tarball_url, no_files):
+    if tarball_url.startswith("file:") and no_files:
+        raise LockfilePackageMetaInvalidError("tarball cannot point to a file, got {}".format(tarball_url))
+    return tarball_url.split("?")[0]
 
 
 def _parse_sky_id_from_tarball_url(tarball_url):
diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
index 52c6ab1b27..69f27470e9 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
@@ -120,21 +120,25 @@ class PnpmPackageManager(BasePackageManager):
 
         return (errors, ins, outs)
 
-    def extract_packages_meta_from_lockfiles(self, lf_paths):
+    def extract_packages_meta_from_lockfiles(self, lf_paths, no_files=False):
         """
         :type lf_paths: iterable of BaseLockfile
         :rtype: iterable of LockfilePackageMeta
         """
         tarballs = set()
+        errors = []
 
         for lf_path in lf_paths:
             try:
-                for pkg in self.load_lockfile(lf_path).get_packages_meta():
+                for pkg in self.load_lockfile(lf_path).get_packages_meta(no_files):
                     if pkg.tarball_path not in tarballs:
                         tarballs.add(pkg.tarball_path)
                         yield pkg
             except Exception as e:
-                raise PackageManagerError("Unable to process lockfile {}: {}".format(lf_path, e))
+                errors.append("{}: {}".format(lf_path, e))
+
+        if errors:
+            raise PackageManagerError("Unable to process some lockfiles:\n{}".format("\n".join(errors)))
 
     def _prepare_workspace(self):
         """
diff --git a/build/plugins/lib/nots/semver/tests/test_version_range.py b/build/plugins/lib/nots/semver/tests/test_version_range.py
index e0833b6dba..eb36d5d598 100644
--- a/build/plugins/lib/nots/semver/tests/test_version_range.py
+++ b/build/plugins/lib/nots/semver/tests/test_version_range.py
@@ -98,7 +98,6 @@ def test_is_satisfied():
     ]
 
     for range_provided, version_provided, expected_result in checklist:
-
         version = Version.from_str(version_provided)
         range = VersionRange.from_str(range_provided)
 
diff --git a/build/plugins/nots.py b/build/plugins/nots.py
index 1421903a37..3d8edcf47b 100644
--- a/build/plugins/nots.py
+++ b/build/plugins/nots.py
@@ -135,9 +135,9 @@ def on_from_npm_lockfiles(unit, *args):
             ymake.report_configure_error("lockfile not found: {}".format(lf_path))
 
     try:
-        for pkg in pm.extract_packages_meta_from_lockfiles(lf_paths):
+        for pkg in pm.extract_packages_meta_from_lockfiles(lf_paths, no_files=True):
             unit.on_from_npm(
-                [pkg.name, pkg.version, pkg.sky_id, pkg.integrity, pkg.integrity_algorithm, pkg.tarball_path]
+                [pkg.tarball_url, pkg.sky_id, pkg.integrity, pkg.integrity_algorithm, pkg.tarball_path]
             )
     except Exception as e:
         if unit.get("TS_RAISE") == "yes":
diff --git a/build/scripts/fetch_from_npm.py b/build/scripts/fetch_from_npm.py
index 3f87c1fefa..e1db347841 100644
--- a/build/scripts/fetch_from_npm.py
+++ b/build/scripts/fetch_from_npm.py
@@ -9,15 +9,14 @@ import sky
 import fetch_from
 
 
-NPM_BASEURL = "http://npm.yandex-team.ru/"
+NPM_BASEURL = "http://npm.yandex-team.ru"
 
 
 def parse_args():
     parser = argparse.ArgumentParser()
     fetch_from.add_common_arguments(parser)
 
-    parser.add_argument("--name", required=True)
-    parser.add_argument("--version", required=True)
+    parser.add_argument("--tarball-url", required=True)
     parser.add_argument("--sky-id", required=True)
     parser.add_argument("--integrity", required=True)
     parser.add_argument("--integrity-algorithm", required=True)
@@ -25,7 +24,7 @@ def parse_args():
     return parser.parse_args()
 
 
-def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, tries=5):
+def fetch(tarball_url, sky_id, integrity, integrity_algorithm, file_name, tries=5):
     """
     :param name: package name
     :type name: str
@@ -50,7 +49,7 @@ def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, trie
     if 'NOTS_FETCH_FROM_SKY' in os.environ and sky.is_avaliable():
         fetcher = lambda: sky.fetch(sky_id, file_name)
     else:
-        fetcher = lambda: _fetch_via_http(name, version, integrity, integrity_algorithm, file_name)
+        fetcher = lambda: _fetch_via_http(tarball_url, integrity, integrity_algorithm, file_name)
 
     fetched_file = None
     exc_info = None
@@ -71,9 +70,10 @@ def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, trie
     return fetched_file
 
 
-def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name):
-    # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1.
-    url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)])
+def _fetch_via_http(tarball_url, integrity, integrity_algorithm, file_name):
+    is_abs_url = tarball_url.startswith("https://") or tarball_url.startswith("http://")
+    url_delim = "" if tarball_url.startswith("/") else "/"
+    url = tarball_url if is_abs_url else NPM_BASEURL + url_delim + tarball_url
 
     hashobj = hashlib.new(integrity_algorithm)
     fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update])
@@ -90,7 +90,7 @@ def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name):
 
 def main(args):
     file_name = os.path.basename(args.copy_to)
-    fetched_file = fetch(args.name, args.version, args.sky_id, args.integrity, args.integrity_algorithm, file_name)
+    fetched_file = fetch(args.tarball_url, args.sky_id, args.integrity, args.integrity_algorithm, file_name)
     fetch_from.process(fetched_file, file_name, args)