feat(FROM_NPM): load packages by tarball url

Раньше FROM_NPM строил урл для скачивания по имени пакета. Изменил это на использование поля tarball, потому что это требуется для интеграции с `@yatool/prebuilder`

3 files changed, 26 insertions, 33 deletions

diff --git a/build/plugins/lib/nots/package_manager/base/lockfile.py b/build/plugins/lib/nots/package_manager/base/lockfile.py
index 1d7cc6ad3e..10cfcc0fc6 100644
--- a/build/plugins/lib/nots/package_manager/base/lockfile.py
+++ b/build/plugins/lib/nots/package_manager/base/lockfile.py
@@ -9,22 +9,24 @@ class LockfilePackageMeta(object):
     Basic struct representing package meta from lockfile.
     """
 
-    __slots__ = ("name", "version", "sky_id", "integrity", "integrity_algorithm", "tarball_path")
+    __slots__ = ("tarball_url", "sky_id", "integrity", "integrity_algorithm", "tarball_path")
 
     @staticmethod
     def from_str(s):
         return LockfilePackageMeta(*s.strip().split(" "))
 
-    def __init__(self, name, version, sky_id, integrity, integrity_algorithm):
-        self.name = name
-        self.version = version
+    def __init__(self, tarball_url, sky_id, integrity, integrity_algorithm):
+        # http://npm.yandex-team.ru/@scope%2fname/-/name-0.0.1.tgz
+        parts = tarball_url.split("/")
+
+        self.tarball_url = tarball_url
         self.sky_id = sky_id
         self.integrity = integrity
         self.integrity_algorithm = integrity_algorithm
-        self.tarball_path = "{}-{}.tgz".format(name, version)
+        self.tarball_path = "/".join(parts[-3:])  # @scope%2fname/-/name-0.0.1.tgz
 
     def to_str(self):
-        return " ".join([self.name, self.version, self.sky_id, self.integrity, self.integrity_algorithm])
+        return " ".join([self.tarball_url, self.sky_id, self.integrity, self.integrity_algorithm])
 
 
 class LockfilePackageMetaInvalidError(RuntimeError):
@@ -61,7 +63,7 @@ class BaseLockfile(object):
         pass
 
     @abstractmethod
-    def get_packages_meta(self):
+    def get_packages_meta(self, no_files):
         pass
 
     @abstractmethod
diff --git a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
index 79c351b7fa..5e55a6f661 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py
@@ -27,14 +27,14 @@ class PnpmLockfile(BaseLockfile):
         with open(path, "w") as f:
             yaml.dump(self.data, f, Dumper=yaml.CSafeDumper)
 
-    def get_packages_meta(self):
+    def get_packages_meta(self, no_files):
         """
         Extracts packages meta from lockfile.
         :rtype: list of LockfilePackageMeta
         """
         packages = self.data.get("packages", {})
 
-        return map(lambda x: _parse_package_meta(*x), iteritems(packages))
+        return map(lambda x: _parse_package_meta(x[0], x[1], no_files), iteritems(packages))
 
     def update_tarball_resolutions(self, fn):
         """
@@ -44,7 +44,7 @@ class PnpmLockfile(BaseLockfile):
         packages = self.data.get("packages", {})
 
         for key, meta in iteritems(packages):
-            meta["resolution"]["tarball"] = fn(_parse_package_meta(key, meta))
+            meta["resolution"]["tarball"] = fn(_parse_package_meta(key, meta, no_files=False))
             packages[key] = meta
 
     def get_importers(self):
@@ -89,7 +89,7 @@ class PnpmLockfile(BaseLockfile):
         self.data["packages"] = packages
 
 
-def _parse_package_meta(key, meta):
+def _parse_package_meta(key, meta, no_files):
     """
     :param key: uniq package key from lockfile
     :type key: string
@@ -98,7 +98,7 @@ def _parse_package_meta(key, meta):
     :rtype: LockfilePackageMetaInvalidError
     """
     try:
-        name, version = _parse_package_key(key)
+        tarball_url = _parse_tarball_url(meta["resolution"]["tarball"], no_files)
         sky_id = _parse_sky_id_from_tarball_url(meta["resolution"]["tarball"])
         integrity_algorithm, integrity = _parse_package_integrity(meta["resolution"]["integrity"])
     except KeyError as e:
@@ -106,26 +106,13 @@ def _parse_package_meta(key, meta):
     except LockfilePackageMetaInvalidError as e:
         raise TypeError("Invalid package meta for key {}, parse error: {}".format(key, e))
 
-    return LockfilePackageMeta(name, version, sky_id, integrity, integrity_algorithm)
+    return LockfilePackageMeta(tarball_url, sky_id, integrity, integrity_algorithm)
 
 
-def _parse_package_key(key):
-    """
-    Returns tuple of scoped package name and version.
-    :param key: package key in format "/({scope}/)?{package_name}/{package_version}(_{peer_dependencies})?"
-    :type key: string
-    :rtype: (str, str)
-    """
-    try:
-        tokens = key.split("/")[1:]
-        version = tokens.pop().split("_", 1)[0]
-
-        if len(tokens) < 1 or len(tokens) > 2:
-            raise TypeError()
-    except (IndexError, TypeError):
-        raise LockfilePackageMetaInvalidError("Invalid package key")
-
-    return ("/".join(tokens), version)
+def _parse_tarball_url(tarball_url, no_files):
+    if tarball_url.startswith("file:") and no_files:
+        raise LockfilePackageMetaInvalidError("tarball cannot point to a file, got {}".format(tarball_url))
+    return tarball_url.split("?")[0]
 
 
 def _parse_sky_id_from_tarball_url(tarball_url):
diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
index 52c6ab1b27..69f27470e9 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
@@ -120,21 +120,25 @@ class PnpmPackageManager(BasePackageManager):
 
         return (errors, ins, outs)
 
-    def extract_packages_meta_from_lockfiles(self, lf_paths):
+    def extract_packages_meta_from_lockfiles(self, lf_paths, no_files=False):
         """
         :type lf_paths: iterable of BaseLockfile
         :rtype: iterable of LockfilePackageMeta
         """
         tarballs = set()
+        errors = []
 
         for lf_path in lf_paths:
             try:
-                for pkg in self.load_lockfile(lf_path).get_packages_meta():
+                for pkg in self.load_lockfile(lf_path).get_packages_meta(no_files):
                     if pkg.tarball_path not in tarballs:
                         tarballs.add(pkg.tarball_path)
                         yield pkg
             except Exception as e:
-                raise PackageManagerError("Unable to process lockfile {}: {}".format(lf_path, e))
+                errors.append("{}: {}".format(lf_path, e))
+
+        if errors:
+            raise PackageManagerError("Unable to process some lockfiles:\n{}".format("\n".join(errors)))
 
     def _prepare_workspace(self):
         """