diff options
| author | dankolesnikov <dankolesnikov@yandex-team.ru> | 2022-02-10 16:51:07 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:51:07 +0300 |
| commit | 69eb1eb80b4c5ddfb3b550e7ce5b10f87f6a7dc3 (patch) | |
| tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
| parent | 98174ab8f0e6dbed8894c8226f62cac4bf36171d (diff) | |
| download | ydb-69eb1eb80b4c5ddfb3b550e7ce5b10f87f6a7dc3.tar.gz | |
Restoring authorship annotation for <dankolesnikov@yandex-team.ru>. Commit 2 of 2.
42 files changed, 2141 insertions, 2141 deletions
diff --git a/build/conf/ts.conf b/build/conf/ts.conf index 0a6310d8b3..8fc221c3ca 100644 --- a/build/conf/ts.conf +++ b/build/conf/ts.conf @@ -1,105 +1,105 @@ -NODEJS_ROOT=$NODEJS_RESOURCE_GLOBAL -NODEJS_BIN=$NODEJS_ROOT/node -PNPM_ROOT=$PNPM_RESOURCE_GLOBAL -PNPM_SCRIPT=$PNPM_ROOT/pnpm.js -TSC_ROOT=$TS_COMPILER_RESOURCE_GLOBAL/typescript -TSC_SCRIPT=$TSC_ROOT/lib/tsc.js -NOTS_TOOL=${tool:"tools/nots"} -NOTS_TOOL_BASE_ARGS=--build-root $ARCADIA_BUILD_ROOT --bindir $BINDIR --curdir $CURDIR --nodejs-bin $NODEJS_BIN - -NPM_CONTRIBS_PATH=devtools/dummy_arcadia/ts/contribs - -### @usage: NPM_CONTRIBS() # internal -### -### Defines special module that provides contrib tarballs from internal npm registry. -### Should be used only with `NODE_MODULES` macro. -### -### @see [FROM_NPM_LOCKFILES()](#macro_FROM_NPM_LOCKFILES) -### @see [NODE_MODULES()](#macro_NODE_MODULES) -module NPM_CONTRIBS: _BASE_UNIT { - .CMD=TOUCH_UNIT - .PEERDIR_POLICY=as_build_from - .FINAL_TARGET=no - .ALLOWED=FROM_NPM_LOCKFILES - .RESTRICTED=PEERDIR - - SET(MODULE_TAG NPM_CONTRIBS) - - MODULE_SUFFIX=.fake - - _BARE_MODULE() -} - -### @usage: FROM_NPM_LOCKFILES(LOCKFILES...) # internal -### -### Defines lockfile list for `NPM_CONTRIBS` module. -### -### @see [NPM_CONTRIBS()](#module_NPM_CONTRIBS) -macro FROM_NPM_LOCKFILES(LOCKFILES...) { - SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS $LOCKFILES) - # See implementation in build/plugins/nots.py - _FROM_NPM_LOCKFILES($LOCKFILES) -} - -### @usage: FROM_NPM(NAME VERSION SKY_ID INTEGRITY INTEGRITY_ALGO TARBALL_PATH) -macro FROM_NPM(NAME, VERSION, SKY_ID, INTEGRITY, INTEGRITY_ALGO, TARBALL_PATH) { +NODEJS_ROOT=$NODEJS_RESOURCE_GLOBAL +NODEJS_BIN=$NODEJS_ROOT/node +PNPM_ROOT=$PNPM_RESOURCE_GLOBAL +PNPM_SCRIPT=$PNPM_ROOT/pnpm.js +TSC_ROOT=$TS_COMPILER_RESOURCE_GLOBAL/typescript +TSC_SCRIPT=$TSC_ROOT/lib/tsc.js +NOTS_TOOL=${tool:"tools/nots"} +NOTS_TOOL_BASE_ARGS=--build-root $ARCADIA_BUILD_ROOT --bindir $BINDIR --curdir $CURDIR --nodejs-bin $NODEJS_BIN + +NPM_CONTRIBS_PATH=devtools/dummy_arcadia/ts/contribs + +### @usage: NPM_CONTRIBS() # internal +### +### Defines special module that provides contrib tarballs from internal npm registry. +### Should be used only with `NODE_MODULES` macro. +### +### @see [FROM_NPM_LOCKFILES()](#macro_FROM_NPM_LOCKFILES) +### @see [NODE_MODULES()](#macro_NODE_MODULES) +module NPM_CONTRIBS: _BASE_UNIT { + .CMD=TOUCH_UNIT + .PEERDIR_POLICY=as_build_from + .FINAL_TARGET=no + .ALLOWED=FROM_NPM_LOCKFILES + .RESTRICTED=PEERDIR + + SET(MODULE_TAG NPM_CONTRIBS) + + MODULE_SUFFIX=.fake + + _BARE_MODULE() +} + +### @usage: FROM_NPM_LOCKFILES(LOCKFILES...) # internal +### +### Defines lockfile list for `NPM_CONTRIBS` module. +### +### @see [NPM_CONTRIBS()](#module_NPM_CONTRIBS) +macro FROM_NPM_LOCKFILES(LOCKFILES...) { + SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS $LOCKFILES) + # See implementation in build/plugins/nots.py + _FROM_NPM_LOCKFILES($LOCKFILES) +} + +### @usage: FROM_NPM(NAME VERSION SKY_ID INTEGRITY INTEGRITY_ALGO TARBALL_PATH) +macro FROM_NPM(NAME, VERSION, SKY_ID, INTEGRITY, INTEGRITY_ALGO, TARBALL_PATH) { .CMD=${cwd:BINDIR} $YMAKE_PYTHON ${input:"build/scripts/fetch_from_npm.py"} ${input;hide:"build/scripts/fetch_from.py"} ${input;hide:"build/scripts/sky.py"} --name $NAME --version $VERSION --sky-id $SKY_ID --integrity $INTEGRITY --integrity-algorithm $INTEGRITY_ALGO --copy-to ${output:TARBALL_PATH} ${requirements;hide:"network:full"} ${kv;hide:"p NP"} ${kv;hide:"pc yellow"} -} - -### @usage: NODE_MODULES() -### -### Materializes `node_modules.tar` bundle according to the module's lockfile. -### -### @see [NPM_CONTRIBS()](#module_NPM_CONTRIBS) -macro NODE_MODULES() { - # See implementation in build/plugins/nots.py -} - -_NODE_MODULES_INPUTS= - -macro _NODE_MODULES(IN{input}[], OUT{output}[]) { - PEERDIR($NPM_CONTRIBS_PATH) - SET(_NODE_MODULES_INPUTS \${input;hide:node_modules.tar}) - +} + +### @usage: NODE_MODULES() +### +### Materializes `node_modules.tar` bundle according to the module's lockfile. +### +### @see [NPM_CONTRIBS()](#module_NPM_CONTRIBS) +macro NODE_MODULES() { + # See implementation in build/plugins/nots.py +} + +_NODE_MODULES_INPUTS= + +macro _NODE_MODULES(IN{input}[], OUT{output}[]) { + PEERDIR($NPM_CONTRIBS_PATH) + SET(_NODE_MODULES_INPUTS \${input;hide:node_modules.tar}) + .CMD=${cwd:BINDIR} $NOTS_TOOL create-node-modules $NOTS_TOOL_BASE_ARGS --pnpm-script $PNPM_SCRIPT --contribs $NPM_CONTRIBS_PATH ${input;hide:IN} ${output;global;hide:OUT} ${kv;hide:"p NM"} ${kv;hide:"pc yellow"} -} - -module _TS_BASE_UNIT: _BASE_UNIT { - SET(MODULE_TAG TS) - SET(PEERDIR_TAGS TS NPM_CONTRIBS) - - PEERDIR(build/platform/nodejs) - PEERDIR(build/external_resources/pnpm) - PEERDIR(build/external_resources/typescript) - - MODULE_SUFFIX=.fake - - _BARE_MODULE() -} - -TS_CONFIG_PATH=tsconfig.json - +} + +module _TS_BASE_UNIT: _BASE_UNIT { + SET(MODULE_TAG TS) + SET(PEERDIR_TAGS TS NPM_CONTRIBS) + + PEERDIR(build/platform/nodejs) + PEERDIR(build/external_resources/pnpm) + PEERDIR(build/external_resources/typescript) + + MODULE_SUFFIX=.fake + + _BARE_MODULE() +} + +TS_CONFIG_PATH=tsconfig.json + TS_COMPILE=${cwd:BINDIR} $NOTS_TOOL compile-ts $NOTS_TOOL_BASE_ARGS --tsc-script $TSC_SCRIPT --config ${input:TS_CONFIG_PATH} $_NODE_MODULES_INPUTS ${output;hide:"package.json"} ${kv;hide:"p FE"} ${kv;hide:"pc yellow"} && $TOUCH_UNIT - -### @usage: TS_LIBRARY([name]) -### -### The TypeScript/JavaScript library module, compiles TypeScript sources to JavaScript. -### Build results are JavaScript files, typings and source mappings (depending on local tsconfig.json settings). -### -### @see [NODE_MODULES()](#macro_NODE_MODULES) -### @example -### -### TS_LIBRARY() -### SRCS(src/index.ts) -### NODE_MODULES() -### END() -### -module TS_LIBRARY : _TS_BASE_UNIT { - .CMD=TS_COMPILE - .FINAL_TARGET=yes - .PEERDIR_POLICY=as_build_from - - SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/${TS_CONFIG_PATH} ${CURDIR}/package.json) - - _TS_CONFIGURE($TS_CONFIG_PATH) -} + +### @usage: TS_LIBRARY([name]) +### +### The TypeScript/JavaScript library module, compiles TypeScript sources to JavaScript. +### Build results are JavaScript files, typings and source mappings (depending on local tsconfig.json settings). +### +### @see [NODE_MODULES()](#macro_NODE_MODULES) +### @example +### +### TS_LIBRARY() +### SRCS(src/index.ts) +### NODE_MODULES() +### END() +### +module TS_LIBRARY : _TS_BASE_UNIT { + .CMD=TS_COMPILE + .FINAL_TARGET=yes + .PEERDIR_POLICY=as_build_from + + SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${CURDIR}/${TS_CONFIG_PATH} ${CURDIR}/package.json) + + _TS_CONFIGURE($TS_CONFIG_PATH) +} diff --git a/build/external_resources/pnpm/readme.md b/build/external_resources/pnpm/readme.md index 9930c11349..1deff50cd7 100644 --- a/build/external_resources/pnpm/readme.md +++ b/build/external_resources/pnpm/readme.md @@ -1,4 +1,4 @@ -# pnpm bundle - -Ресурс должен быть tar-архивом с собранным в один файл pnpm'ом – `pnpm.js`. -Собранный файл можно взять в `dist/` пакета `pnpm`. Зависимости для актуальной версии (5.18.9) не требуются. +# pnpm bundle + +Ресурс должен быть tar-архивом с собранным в один файл pnpm'ом – `pnpm.js`. +Собранный файл можно взять в `dist/` пакета `pnpm`. Зависимости для актуальной версии (5.18.9) не требуются. diff --git a/build/external_resources/pnpm/ya.make b/build/external_resources/pnpm/ya.make index daaeaadc9a..0e6fd426a1 100644 --- a/build/external_resources/pnpm/ya.make +++ b/build/external_resources/pnpm/ya.make @@ -1,8 +1,8 @@ -RESOURCES_LIBRARY() - -OWNER(dankolesnikov) - -# pnpm 5.18.9 -DECLARE_EXTERNAL_RESOURCE(PNPM sbr:2197487733) - -END() +RESOURCES_LIBRARY() + +OWNER(dankolesnikov) + +# pnpm 5.18.9 +DECLARE_EXTERNAL_RESOURCE(PNPM sbr:2197487733) + +END() diff --git a/build/external_resources/typescript/readme.md b/build/external_resources/typescript/readme.md index ae8a3a66fe..dba0d171f2 100644 --- a/build/external_resources/typescript/readme.md +++ b/build/external_resources/typescript/readme.md @@ -1,12 +1,12 @@ -# typescript bundle - -Ресурс должен быть tar-архивом с собранным пакетом `typescript`. -В tar должны быть включены тайпинги для поддерживаемых target/lib и `tsc.js`. Структура: - -``` -lib/ - tsc.js - lib.*.d.td -``` - -Зависимости для актуальной версии не требуются. +# typescript bundle + +Ресурс должен быть tar-архивом с собранным пакетом `typescript`. +В tar должны быть включены тайпинги для поддерживаемых target/lib и `tsc.js`. Структура: + +``` +lib/ + tsc.js + lib.*.d.td +``` + +Зависимости для актуальной версии не требуются. diff --git a/build/external_resources/typescript/ya.make b/build/external_resources/typescript/ya.make index 4d29ba16cd..e5141556e4 100644 --- a/build/external_resources/typescript/ya.make +++ b/build/external_resources/typescript/ya.make @@ -1,8 +1,8 @@ -RESOURCES_LIBRARY() - -OWNER(dankolesnikov) - -# typescript 4.1.3 -DECLARE_EXTERNAL_RESOURCE(TS_COMPILER sbr:2202008231) - -END() +RESOURCES_LIBRARY() + +OWNER(dankolesnikov) + +# typescript 4.1.3 +DECLARE_EXTERNAL_RESOURCE(TS_COMPILER sbr:2202008231) + +END() diff --git a/build/external_resources/ya.make b/build/external_resources/ya.make index 7ca26d0a06..ba2ae38588 100644 --- a/build/external_resources/ya.make +++ b/build/external_resources/ya.make @@ -34,8 +34,8 @@ IF (OS_DARWIN OR OS_LINUX OR OS_WINDOWS) go_fake_xcrun go_tools goyndexer - pnpm - typescript + pnpm + typescript ymake yolint ) diff --git a/build/platform/nodejs/readme.md b/build/platform/nodejs/readme.md index 9b7cc83605..92fbcfe699 100644 --- a/build/platform/nodejs/readme.md +++ b/build/platform/nodejs/readme.md @@ -1,5 +1,5 @@ -# Node.js bundle - -В бандле находится аркадийная Node.js (`contrib/libs/nodejs_12`) для Linux и Darwin. - -Ресурс должен быть tar-архивом с исполнимым бинарём `node`. +# Node.js bundle + +В бандле находится аркадийная Node.js (`contrib/libs/nodejs_12`) для Linux и Darwin. + +Ресурс должен быть tar-архивом с исполнимым бинарём `node`. diff --git a/build/platform/nodejs/resources.inc b/build/platform/nodejs/resources.inc index 3d239a63db..002466daeb 100644 --- a/build/platform/nodejs/resources.inc +++ b/build/platform/nodejs/resources.inc @@ -1,2 +1,2 @@ -SET(NODEJS_12_18_4_LINUX sbr:2197482931) -SET(NODEJS_12_18_4_DARWIN sbr:2197413902) +SET(NODEJS_12_18_4_LINUX sbr:2197482931) +SET(NODEJS_12_18_4_DARWIN sbr:2197413902) diff --git a/build/platform/nodejs/ya.make b/build/platform/nodejs/ya.make index f2f0526ac6..c872eca0d8 100644 --- a/build/platform/nodejs/ya.make +++ b/build/platform/nodejs/ya.make @@ -1,17 +1,17 @@ -RESOURCES_LIBRARY() - -OWNER(dankolesnikov) - -INCLUDE(${ARCADIA_ROOT}/build/platform/nodejs/resources.inc) - -IF (NOT HOST_OS_LINUX AND NOT HOST_OS_DARWIN) - MESSAGE(FATAL_ERROR Unsupported platform for Nodejs) -ENDIF() - -DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE( - NODEJS - ${NODEJS_12_18_4_LINUX} FOR LINUX - ${NODEJS_12_18_4_DARWIN} FOR DARWIN -) - -END() +RESOURCES_LIBRARY() + +OWNER(dankolesnikov) + +INCLUDE(${ARCADIA_ROOT}/build/platform/nodejs/resources.inc) + +IF (NOT HOST_OS_LINUX AND NOT HOST_OS_DARWIN) + MESSAGE(FATAL_ERROR Unsupported platform for Nodejs) +ENDIF() + +DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE( + NODEJS + ${NODEJS_12_18_4_LINUX} FOR LINUX + ${NODEJS_12_18_4_DARWIN} FOR DARWIN +) + +END() diff --git a/build/plugins/_common.py b/build/plugins/_common.py index 01604cf3c6..2f831a94db 100644 --- a/build/plugins/_common.py +++ b/build/plugins/_common.py @@ -195,7 +195,7 @@ def strip_roots(path): if path.startswith(prefix): return path[len(prefix):] return path - - -def to_yesno(x): - return "yes" if x else "no" + + +def to_yesno(x): + return "yes" if x else "no" diff --git a/build/plugins/lib/nots/package_manager/__init__.py b/build/plugins/lib/nots/package_manager/__init__.py index 4fc30d2525..52bf62644c 100644 --- a/build/plugins/lib/nots/package_manager/__init__.py +++ b/build/plugins/lib/nots/package_manager/__init__.py @@ -1,9 +1,9 @@ -from .pnpm import PnpmPackageManager -from .base import constants - - -manager = PnpmPackageManager - -__all__ = [ - "constants", -] +from .pnpm import PnpmPackageManager +from .base import constants + + +manager = PnpmPackageManager + +__all__ = [ + "constants", +] diff --git a/build/plugins/lib/nots/package_manager/base/__init__.py b/build/plugins/lib/nots/package_manager/base/__init__.py index b7f6729eec..1b55fe3f56 100644 --- a/build/plugins/lib/nots/package_manager/base/__init__.py +++ b/build/plugins/lib/nots/package_manager/base/__init__.py @@ -1,11 +1,11 @@ -from . import constants -from .lockfile import BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError -from .package_json import PackageJson -from .package_manager import BasePackageManager, PackageManagerError, PackageManagerCommandError - -__all__ = [ - "constants", - "BaseLockfile", "LockfilePackageMeta", "LockfilePackageMetaInvalidError", - "BasePackageManager", "PackageManagerError", "PackageManagerCommandError", - "PackageJson", -] +from . import constants +from .lockfile import BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError +from .package_json import PackageJson +from .package_manager import BasePackageManager, PackageManagerError, PackageManagerCommandError + +__all__ = [ + "constants", + "BaseLockfile", "LockfilePackageMeta", "LockfilePackageMetaInvalidError", + "BasePackageManager", "PackageManagerError", "PackageManagerCommandError", + "PackageJson", +] diff --git a/build/plugins/lib/nots/package_manager/base/constants.py b/build/plugins/lib/nots/package_manager/base/constants.py index 89bfe8a42e..0b9fcb76af 100644 --- a/build/plugins/lib/nots/package_manager/base/constants.py +++ b/build/plugins/lib/nots/package_manager/base/constants.py @@ -1,5 +1,5 @@ -PACKAGE_JSON_FILENAME = "package.json" -NODE_MODULES_BUNDLE_FILENAME = "node_modules.tar" -NPM_REGISTRY_URL = "http://npm.yandex-team.ru" -PNPM_WS_FILENAME = "pnpm-workspace.yaml" -PNPM_LOCKFILE_FILENAME = "pnpm-lock.yaml" +PACKAGE_JSON_FILENAME = "package.json" +NODE_MODULES_BUNDLE_FILENAME = "node_modules.tar" +NPM_REGISTRY_URL = "http://npm.yandex-team.ru" +PNPM_WS_FILENAME = "pnpm-workspace.yaml" +PNPM_LOCKFILE_FILENAME = "pnpm-lock.yaml" diff --git a/build/plugins/lib/nots/package_manager/base/lockfile.py b/build/plugins/lib/nots/package_manager/base/lockfile.py index 8a377045ed..9b9c0be954 100644 --- a/build/plugins/lib/nots/package_manager/base/lockfile.py +++ b/build/plugins/lib/nots/package_manager/base/lockfile.py @@ -1,68 +1,68 @@ -import os - -from abc import ABCMeta, abstractmethod -from six import add_metaclass - - -class LockfilePackageMeta(object): - """ - Basic struct representing package meta from lockfile. - """ - __slots__ = ("name", "version", "sky_id", "integrity", "integrity_algorithm", "tarball_path") - - @staticmethod - def from_str(s): - return LockfilePackageMeta(*s.strip().split(" ")) - - def __init__(self, name, version, sky_id, integrity, integrity_algorithm): - self.name = name - self.version = version - self.sky_id = sky_id - self.integrity = integrity - self.integrity_algorithm = integrity_algorithm - self.tarball_path = "{}-{}.tgz".format(name, version) - - def to_str(self): - return " ".join([self.name, self.version, self.sky_id, self.integrity, self.integrity_algorithm]) - - -class LockfilePackageMetaInvalidError(RuntimeError): - pass - - -@add_metaclass(ABCMeta) -class BaseLockfile(object): - @classmethod - def load(cls, path): - """ - :param path: lockfile path - :type path: str - :rtype: BaseLockfile - """ - pj = cls(path) - pj.read() - - return pj - - def __init__(self, path): - if not os.path.isabs(path): - raise TypeError("Absolute path required, given: {}".format(path)) - - self.path = path - self.data = None - - @abstractmethod - def read(self): - pass - - @abstractmethod - def write(self, path=None): - pass - - @abstractmethod - def get_packages_meta(self): - pass - - @abstractmethod - def update_tarball_resolutions(self, fn): - pass +import os + +from abc import ABCMeta, abstractmethod +from six import add_metaclass + + +class LockfilePackageMeta(object): + """ + Basic struct representing package meta from lockfile. + """ + __slots__ = ("name", "version", "sky_id", "integrity", "integrity_algorithm", "tarball_path") + + @staticmethod + def from_str(s): + return LockfilePackageMeta(*s.strip().split(" ")) + + def __init__(self, name, version, sky_id, integrity, integrity_algorithm): + self.name = name + self.version = version + self.sky_id = sky_id + self.integrity = integrity + self.integrity_algorithm = integrity_algorithm + self.tarball_path = "{}-{}.tgz".format(name, version) + + def to_str(self): + return " ".join([self.name, self.version, self.sky_id, self.integrity, self.integrity_algorithm]) + + +class LockfilePackageMetaInvalidError(RuntimeError): + pass + + +@add_metaclass(ABCMeta) +class BaseLockfile(object): + @classmethod + def load(cls, path): + """ + :param path: lockfile path + :type path: str + :rtype: BaseLockfile + """ + pj = cls(path) + pj.read() + + return pj + + def __init__(self, path): + if not os.path.isabs(path): + raise TypeError("Absolute path required, given: {}".format(path)) + + self.path = path + self.data = None + + @abstractmethod + def read(self): + pass + + @abstractmethod + def write(self, path=None): + pass + + @abstractmethod + def get_packages_meta(self): + pass + + @abstractmethod + def update_tarball_resolutions(self, fn): + pass diff --git a/build/plugins/lib/nots/package_manager/base/package_json.py b/build/plugins/lib/nots/package_manager/base/package_json.py index f9c1caf368..3d0bf3238e 100644 --- a/build/plugins/lib/nots/package_manager/base/package_json.py +++ b/build/plugins/lib/nots/package_manager/base/package_json.py @@ -1,113 +1,113 @@ -import os -import json - -from six import iteritems - -from . import constants - - -class PackageJsonWorkspaceError(RuntimeError): - pass - - -class PackageJson(object): - DEP_KEY = "dependencies" - DEV_DEP_KEY = "devDependencies" - PEER_DEP_KEY = "peerDependencies" - OPT_DEP_KEY = "optionalDependencies" - DEP_KEYS = (DEP_KEY, DEV_DEP_KEY, PEER_DEP_KEY, OPT_DEP_KEY) - - WORKSPACE_SCHEMA = "workspace:" - - @classmethod - def load(cls, path): - """ - :param path: package.json path - :type path: str - :rtype: PackageJson - """ - pj = cls(path) - pj.read() - - return pj - - def __init__(self, path): - if not os.path.isabs(path): - raise TypeError("Absolute path required, given: {}".format(path)) - - self.path = path - self.data = None - - def read(self): - with open(self.path) as f: - self.data = json.load(f) - - def get_name(self): - return self.data.get("name") - - def get_workspace_dep_paths(self): - """ - :return: Workspace dependencies. - :rtype: list of (str, str) - """ - dep_paths = [] - schema = self.WORKSPACE_SCHEMA - schema_len = len(schema) - - for deps in map(lambda x: self.data.get(x), self.DEP_KEYS): - if not deps: - continue - - for name, spec in iteritems(deps): - if not spec.startswith(schema): - continue - - spec_path = spec[schema_len:] - if not (spec_path.startswith(".") or spec_path.startswith("..")): - raise PackageJsonWorkspaceError( - "Expected relative path specifier for workspace dependency, but got '{}' for {} in {}".format(spec, name, self.path)) - - dep_paths.append((name, spec_path)) - - return dep_paths - - def get_workspace_deps(self): - """ - :rtype: list of PackageJson - """ - ws_deps = [] - pj_dir = os.path.dirname(self.path) - - for (name, rel_path) in self.get_workspace_dep_paths(): - dep_path = os.path.normpath(os.path.join(pj_dir, rel_path)) - dep_pj = PackageJson.load(os.path.join(dep_path, constants.PACKAGE_JSON_FILENAME)) - - if name != dep_pj.get_name(): - raise PackageJsonWorkspaceError( - "Workspace dependency name mismatch, found '{}' instead of '{}' in {}".format(name, dep_pj.get_name(), self.path)) - - ws_deps.append(dep_pj) - - return ws_deps - - def get_workspace_map(self): - """ - :return: Absolute paths of workspace dependencies (including transitive) mapped to package.json and depth. - :rtype: dict of (PackageJson, int) - """ - ws_deps = {} - # list of (pj, depth) - pj_queue = [(self, 0)] - - while len(pj_queue): - (pj, depth) = pj_queue.pop() - pj_dir = os.path.dirname(pj.path) - if pj_dir in ws_deps: - continue - - ws_deps[pj_dir] = (pj, depth) - - for dep_pj in pj.get_workspace_deps(): - pj_queue.append((dep_pj, depth + 1)) - - return ws_deps +import os +import json + +from six import iteritems + +from . import constants + + +class PackageJsonWorkspaceError(RuntimeError): + pass + + +class PackageJson(object): + DEP_KEY = "dependencies" + DEV_DEP_KEY = "devDependencies" + PEER_DEP_KEY = "peerDependencies" + OPT_DEP_KEY = "optionalDependencies" + DEP_KEYS = (DEP_KEY, DEV_DEP_KEY, PEER_DEP_KEY, OPT_DEP_KEY) + + WORKSPACE_SCHEMA = "workspace:" + + @classmethod + def load(cls, path): + """ + :param path: package.json path + :type path: str + :rtype: PackageJson + """ + pj = cls(path) + pj.read() + + return pj + + def __init__(self, path): + if not os.path.isabs(path): + raise TypeError("Absolute path required, given: {}".format(path)) + + self.path = path + self.data = None + + def read(self): + with open(self.path) as f: + self.data = json.load(f) + + def get_name(self): + return self.data.get("name") + + def get_workspace_dep_paths(self): + """ + :return: Workspace dependencies. + :rtype: list of (str, str) + """ + dep_paths = [] + schema = self.WORKSPACE_SCHEMA + schema_len = len(schema) + + for deps in map(lambda x: self.data.get(x), self.DEP_KEYS): + if not deps: + continue + + for name, spec in iteritems(deps): + if not spec.startswith(schema): + continue + + spec_path = spec[schema_len:] + if not (spec_path.startswith(".") or spec_path.startswith("..")): + raise PackageJsonWorkspaceError( + "Expected relative path specifier for workspace dependency, but got '{}' for {} in {}".format(spec, name, self.path)) + + dep_paths.append((name, spec_path)) + + return dep_paths + + def get_workspace_deps(self): + """ + :rtype: list of PackageJson + """ + ws_deps = [] + pj_dir = os.path.dirname(self.path) + + for (name, rel_path) in self.get_workspace_dep_paths(): + dep_path = os.path.normpath(os.path.join(pj_dir, rel_path)) + dep_pj = PackageJson.load(os.path.join(dep_path, constants.PACKAGE_JSON_FILENAME)) + + if name != dep_pj.get_name(): + raise PackageJsonWorkspaceError( + "Workspace dependency name mismatch, found '{}' instead of '{}' in {}".format(name, dep_pj.get_name(), self.path)) + + ws_deps.append(dep_pj) + + return ws_deps + + def get_workspace_map(self): + """ + :return: Absolute paths of workspace dependencies (including transitive) mapped to package.json and depth. + :rtype: dict of (PackageJson, int) + """ + ws_deps = {} + # list of (pj, depth) + pj_queue = [(self, 0)] + + while len(pj_queue): + (pj, depth) = pj_queue.pop() + pj_dir = os.path.dirname(pj.path) + if pj_dir in ws_deps: + continue + + ws_deps[pj_dir] = (pj, depth) + + for dep_pj in pj.get_workspace_deps(): + pj_queue.append((dep_pj, depth + 1)) + + return ws_deps diff --git a/build/plugins/lib/nots/package_manager/base/package_manager.py b/build/plugins/lib/nots/package_manager/base/package_manager.py index 9c0fcb8c34..0de9d8acc3 100644 --- a/build/plugins/lib/nots/package_manager/base/package_manager.py +++ b/build/plugins/lib/nots/package_manager/base/package_manager.py @@ -1,108 +1,108 @@ -import os -import sys -import subprocess -import tarfile - -from abc import ABCMeta, abstractmethod -from six import add_metaclass - -from . import constants - - -class PackageManagerError(RuntimeError): - pass - - -class PackageManagerCommandError(PackageManagerError): - def __init__(self, cmd, code, stdout, stderr): - self.cmd = cmd - self.code = code - self.stdout = stdout - self.stderr = stderr - - msg = "package manager exited with code {} while running {}:\n{}\n{}".format(code, cmd, stdout, stderr) - super(PackageManagerCommandError, self).__init__(msg) - - -@add_metaclass(ABCMeta) -class BasePackageManager(object): - def __init__(self, build_root, build_path, sources_path, nodejs_bin_path, script_path, contribs_path): - self.module_path = build_path[len(build_root) + 1:] - self.build_path = build_path - self.sources_path = sources_path - self.build_root = build_root - self.sources_root = sources_path[:-len(self.module_path) - 1] - self.nodejs_bin_path = nodejs_bin_path - self.script_path = script_path - self.contribs_path = contribs_path - - @abstractmethod - def install(self): - pass - - @abstractmethod - def get_peer_paths_from_package_json(self): - pass - - @abstractmethod - def calc_node_modules_inouts(self): - pass - - @abstractmethod - def extract_packages_meta_from_lockfiles(self, lf_paths): - pass - - def create_node_modules_bundle(self, path): - """ - Creates tarball from the node_modules directory contents. - :param path: tarball path - :type path: str - """ - with tarfile.open(path, "w") as tf: - tf.add(self._nm_path(), arcname=".") - - def _exec_command(self, args, include_defaults=True): - if not self.nodejs_bin_path: - raise PackageManagerError("Unable to execute command: nodejs_bin_path is not configured") - - cmd = [self.nodejs_bin_path, self.script_path] + args + (self._get_default_options() if include_defaults else []) - p = subprocess.Popen( - cmd, - cwd=self.build_path, - stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout, stderr = p.communicate() - - if p.returncode != 0: - self._dump_debug_log() - - raise PackageManagerCommandError(cmd, p.returncode, stdout.decode("utf-8"), stderr.decode("utf-8")) - - def _nm_path(self, *parts): - return os.path.join(self.build_path, "node_modules", *parts) - - def _contrib_tarball_path(self, pkg): - return os.path.join(self.contribs_path, pkg.tarball_path) - - def _contrib_tarball_url(self, pkg): - return "file:" + self._contrib_tarball_path(pkg) - - def _get_default_options(self): - return ["--registry", constants.NPM_REGISTRY_URL] - - def _get_debug_log_path(self): - return None - - def _dump_debug_log(self): - log_path = self._get_debug_log_path() - - if not log_path: - return - - try: - with open(log_path) as f: - sys.stderr.write("Package manager log {}:\n{}\n".format(log_path, f.read())) - except: - sys.stderr.write("Failed to dump package manager log {}.\n".format(log_path)) +import os +import sys +import subprocess +import tarfile + +from abc import ABCMeta, abstractmethod +from six import add_metaclass + +from . import constants + + +class PackageManagerError(RuntimeError): + pass + + +class PackageManagerCommandError(PackageManagerError): + def __init__(self, cmd, code, stdout, stderr): + self.cmd = cmd + self.code = code + self.stdout = stdout + self.stderr = stderr + + msg = "package manager exited with code {} while running {}:\n{}\n{}".format(code, cmd, stdout, stderr) + super(PackageManagerCommandError, self).__init__(msg) + + +@add_metaclass(ABCMeta) +class BasePackageManager(object): + def __init__(self, build_root, build_path, sources_path, nodejs_bin_path, script_path, contribs_path): + self.module_path = build_path[len(build_root) + 1:] + self.build_path = build_path + self.sources_path = sources_path + self.build_root = build_root + self.sources_root = sources_path[:-len(self.module_path) - 1] + self.nodejs_bin_path = nodejs_bin_path + self.script_path = script_path + self.contribs_path = contribs_path + + @abstractmethod + def install(self): + pass + + @abstractmethod + def get_peer_paths_from_package_json(self): + pass + + @abstractmethod + def calc_node_modules_inouts(self): + pass + + @abstractmethod + def extract_packages_meta_from_lockfiles(self, lf_paths): + pass + + def create_node_modules_bundle(self, path): + """ + Creates tarball from the node_modules directory contents. + :param path: tarball path + :type path: str + """ + with tarfile.open(path, "w") as tf: + tf.add(self._nm_path(), arcname=".") + + def _exec_command(self, args, include_defaults=True): + if not self.nodejs_bin_path: + raise PackageManagerError("Unable to execute command: nodejs_bin_path is not configured") + + cmd = [self.nodejs_bin_path, self.script_path] + args + (self._get_default_options() if include_defaults else []) + p = subprocess.Popen( + cmd, + cwd=self.build_path, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + + if p.returncode != 0: + self._dump_debug_log() + + raise PackageManagerCommandError(cmd, p.returncode, stdout.decode("utf-8"), stderr.decode("utf-8")) + + def _nm_path(self, *parts): + return os.path.join(self.build_path, "node_modules", *parts) + + def _contrib_tarball_path(self, pkg): + return os.path.join(self.contribs_path, pkg.tarball_path) + + def _contrib_tarball_url(self, pkg): + return "file:" + self._contrib_tarball_path(pkg) + + def _get_default_options(self): + return ["--registry", constants.NPM_REGISTRY_URL] + + def _get_debug_log_path(self): + return None + + def _dump_debug_log(self): + log_path = self._get_debug_log_path() + + if not log_path: + return + + try: + with open(log_path) as f: + sys.stderr.write("Package manager log {}:\n{}\n".format(log_path, f.read())) + except: + sys.stderr.write("Failed to dump package manager log {}.\n".format(log_path)) diff --git a/build/plugins/lib/nots/package_manager/base/tests/package_json.py b/build/plugins/lib/nots/package_manager/base/tests/package_json.py index 64a7841e50..3657e581bc 100644 --- a/build/plugins/lib/nots/package_manager/base/tests/package_json.py +++ b/build/plugins/lib/nots/package_manager/base/tests/package_json.py @@ -1,114 +1,114 @@ -import os -import pytest - -from build.plugins.lib.nots.package_manager.base.package_json import PackageJson, PackageJsonWorkspaceError - - -def test_get_workspace_dep_paths_ok(): - pj = PackageJson("/packages/foo/package.json") - pj.data = { - "dependencies": { - "@yandex-int/bar": "workspace:../bar", - }, - "devDependencies": { - "@yandex-int/baz": "workspace:../baz", - }, - } - - ws_dep_paths = pj.get_workspace_dep_paths() - - assert ws_dep_paths == [ - ("@yandex-int/bar", "../bar"), - ("@yandex-int/baz", "../baz"), - ] - - -def test_get_workspace_dep_paths_invalid_path(): - pj = PackageJson("/packages/foo/package.json") - pj.data = { - "dependencies": { - "@yandex-int/bar": "workspace:*", - }, - } - - with pytest.raises(PackageJsonWorkspaceError) as e: - pj.get_workspace_dep_paths() - - assert str(e.value) == "Expected relative path specifier for workspace dependency, but got 'workspace:*' for @yandex-int/bar in /packages/foo/package.json" - - -def test_get_workspace_deps_ok(): - pj = PackageJson("/packages/foo/package.json") - pj.data = { - "dependencies": { - "@yandex-int/bar": "workspace:../bar", - }, - "devDependencies": { - "@yandex-int/baz": "workspace:../baz", - }, - } - - def load_mock(cls, path): - p = PackageJson(path) - p.data = { - "name": "@yandex-int/{}".format(os.path.basename(os.path.dirname(path))), - } - return p - PackageJson.load = classmethod(load_mock) - - ws_deps = pj.get_workspace_deps() - - assert len(ws_deps) == 2 - assert ws_deps[0].path == "/packages/bar/package.json" - assert ws_deps[1].path == "/packages/baz/package.json" - - -def test_get_workspace_deps_with_wrong_name(): - pj = PackageJson("/packages/foo/package.json") - pj.data = { - "dependencies": { - "@yandex-int/bar": "workspace:../bar", - }, - } - - def load_mock(cls, path): - p = PackageJson(path) - p.data = { - "name": "@shouldbe/{}".format(os.path.basename(os.path.dirname(path))), - } - return p - PackageJson.load = classmethod(load_mock) - - with pytest.raises(PackageJsonWorkspaceError) as e: - pj.get_workspace_deps() - - assert str(e.value) == "Workspace dependency name mismatch, found '@yandex-int/bar' instead of '@shouldbe/bar' in /packages/foo/package.json" - - -def test_get_workspace_map_ok(): - pj = PackageJson("/packages/foo/package.json") - pj.data = { - "dependencies": { - "@yandex-int/bar": "workspace:../bar", - }, - } - - def load_mock(cls, path): - name = os.path.basename(os.path.dirname(path)) - p = PackageJson(path) - p.data = { - "name": "@yandex-int/{}".format(name), - "dependencies": ({"@yandex-int/qux": "workspace:../qux"} if name == "bar" else {}), - } - return p - PackageJson.load = classmethod(load_mock) - - ws_map = pj.get_workspace_map() - - assert len(ws_map) == 3 - assert ws_map["/packages/foo"][0].path == "/packages/foo/package.json" - assert ws_map["/packages/foo"][1] == 0 - assert ws_map["/packages/bar"][0].path == "/packages/bar/package.json" - assert ws_map["/packages/bar"][1] == 1 - assert ws_map["/packages/qux"][0].path == "/packages/qux/package.json" - assert ws_map["/packages/qux"][1] == 2 +import os +import pytest + +from build.plugins.lib.nots.package_manager.base.package_json import PackageJson, PackageJsonWorkspaceError + + +def test_get_workspace_dep_paths_ok(): + pj = PackageJson("/packages/foo/package.json") + pj.data = { + "dependencies": { + "@yandex-int/bar": "workspace:../bar", + }, + "devDependencies": { + "@yandex-int/baz": "workspace:../baz", + }, + } + + ws_dep_paths = pj.get_workspace_dep_paths() + + assert ws_dep_paths == [ + ("@yandex-int/bar", "../bar"), + ("@yandex-int/baz", "../baz"), + ] + + +def test_get_workspace_dep_paths_invalid_path(): + pj = PackageJson("/packages/foo/package.json") + pj.data = { + "dependencies": { + "@yandex-int/bar": "workspace:*", + }, + } + + with pytest.raises(PackageJsonWorkspaceError) as e: + pj.get_workspace_dep_paths() + + assert str(e.value) == "Expected relative path specifier for workspace dependency, but got 'workspace:*' for @yandex-int/bar in /packages/foo/package.json" + + +def test_get_workspace_deps_ok(): + pj = PackageJson("/packages/foo/package.json") + pj.data = { + "dependencies": { + "@yandex-int/bar": "workspace:../bar", + }, + "devDependencies": { + "@yandex-int/baz": "workspace:../baz", + }, + } + + def load_mock(cls, path): + p = PackageJson(path) + p.data = { + "name": "@yandex-int/{}".format(os.path.basename(os.path.dirname(path))), + } + return p + PackageJson.load = classmethod(load_mock) + + ws_deps = pj.get_workspace_deps() + + assert len(ws_deps) == 2 + assert ws_deps[0].path == "/packages/bar/package.json" + assert ws_deps[1].path == "/packages/baz/package.json" + + +def test_get_workspace_deps_with_wrong_name(): + pj = PackageJson("/packages/foo/package.json") + pj.data = { + "dependencies": { + "@yandex-int/bar": "workspace:../bar", + }, + } + + def load_mock(cls, path): + p = PackageJson(path) + p.data = { + "name": "@shouldbe/{}".format(os.path.basename(os.path.dirname(path))), + } + return p + PackageJson.load = classmethod(load_mock) + + with pytest.raises(PackageJsonWorkspaceError) as e: + pj.get_workspace_deps() + + assert str(e.value) == "Workspace dependency name mismatch, found '@yandex-int/bar' instead of '@shouldbe/bar' in /packages/foo/package.json" + + +def test_get_workspace_map_ok(): + pj = PackageJson("/packages/foo/package.json") + pj.data = { + "dependencies": { + "@yandex-int/bar": "workspace:../bar", + }, + } + + def load_mock(cls, path): + name = os.path.basename(os.path.dirname(path)) + p = PackageJson(path) + p.data = { + "name": "@yandex-int/{}".format(name), + "dependencies": ({"@yandex-int/qux": "workspace:../qux"} if name == "bar" else {}), + } + return p + PackageJson.load = classmethod(load_mock) + + ws_map = pj.get_workspace_map() + + assert len(ws_map) == 3 + assert ws_map["/packages/foo"][0].path == "/packages/foo/package.json" + assert ws_map["/packages/foo"][1] == 0 + assert ws_map["/packages/bar"][0].path == "/packages/bar/package.json" + assert ws_map["/packages/bar"][1] == 1 + assert ws_map["/packages/qux"][0].path == "/packages/qux/package.json" + assert ws_map["/packages/qux"][1] == 2 diff --git a/build/plugins/lib/nots/package_manager/base/tests/ya.make b/build/plugins/lib/nots/package_manager/base/tests/ya.make index d1d9042d21..1968fac42e 100644 --- a/build/plugins/lib/nots/package_manager/base/tests/ya.make +++ b/build/plugins/lib/nots/package_manager/base/tests/ya.make @@ -1,13 +1,13 @@ -PY23_TEST() - -OWNER(dankolesnikov) - -TEST_SRCS( - package_json.py -) - -PEERDIR( - build/plugins/lib/nots/package_manager/base -) - -END() +PY23_TEST() + +OWNER(dankolesnikov) + +TEST_SRCS( + package_json.py +) + +PEERDIR( + build/plugins/lib/nots/package_manager/base +) + +END() diff --git a/build/plugins/lib/nots/package_manager/base/ya.make b/build/plugins/lib/nots/package_manager/base/ya.make index 4b00ef4d85..aa73cfbe25 100644 --- a/build/plugins/lib/nots/package_manager/base/ya.make +++ b/build/plugins/lib/nots/package_manager/base/ya.make @@ -1,21 +1,21 @@ -PY23_LIBRARY() - -OWNER(dankolesnikov) - -PY_SRCS( - __init__.py - constants.py - lockfile.py - package_json.py - package_manager.py -) - -PEERDIR( - contrib/python/six -) - -END() - -RECURSE_FOR_TESTS( - tests -) +PY23_LIBRARY() + +OWNER(dankolesnikov) + +PY_SRCS( + __init__.py + constants.py + lockfile.py + package_json.py + package_manager.py +) + +PEERDIR( + contrib/python/six +) + +END() + +RECURSE_FOR_TESTS( + tests +) diff --git a/build/plugins/lib/nots/package_manager/pnpm/__init__.py b/build/plugins/lib/nots/package_manager/pnpm/__init__.py index 2db11bd080..4b8f0d0e92 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/__init__.py +++ b/build/plugins/lib/nots/package_manager/pnpm/__init__.py @@ -1,9 +1,9 @@ -from .lockfile import PnpmLockfile -from .package_manager import PnpmPackageManager -from .workspace import PnpmWorkspace - -__all__ = [ - "PnpmLockfile", - "PnpmPackageManager", - "PnpmWorkspace", -] +from .lockfile import PnpmLockfile +from .package_manager import PnpmPackageManager +from .workspace import PnpmWorkspace + +__all__ = [ + "PnpmLockfile", + "PnpmPackageManager", + "PnpmWorkspace", +] diff --git a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py index 8bfb2beda1..1c09f96432 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/lockfile.py +++ b/build/plugins/lib/nots/package_manager/pnpm/lockfile.py @@ -1,163 +1,163 @@ -import base64 -import binascii -import yaml -import os - -from six.moves.urllib import parse as urlparse -from six import iteritems - -from ..base import PackageJson, BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError - - -class PnpmLockfile(BaseLockfile): - IMPORTER_KEYS = PackageJson.DEP_KEYS + ("specifiers",) - - def read(self): - with open(self.path, "r") as f: - self.data = yaml.load(f, Loader=yaml.CSafeLoader) - - def write(self, path=None): - """ - :param path: path to store lockfile, defaults to original path - :type path: str - """ - if path is None: - path = self.path - - with open(path, "w") as f: - yaml.dump(self.data, f, Dumper=yaml.CSafeDumper) - - def get_packages_meta(self): - """ - Extracts packages meta from lockfile. - :rtype: list of LockfilePackageMeta - """ - packages = self.data.get("packages", {}) - - return map(lambda x: _parse_package_meta(*x), iteritems(packages)) - - def update_tarball_resolutions(self, fn): - """ - :param fn: maps `LockfilePackageMeta` instance to new `resolution.tarball` value - :type fn: lambda - """ - packages = self.data.get("packages", {}) - - for key, meta in iteritems(packages): - meta["resolution"]["tarball"] = fn(_parse_package_meta(key, meta)) - packages[key] = meta - - def get_importers(self): - """ - Returns "importers" section from the lockfile or creates similar structure from "dependencies" and "specifiers". - :rtype: dict of dict of dict of str - """ - importers = self.data.get("importers") - if importers is not None: - return importers - - importer = {k: self.data[k] for k in self.IMPORTER_KEYS if k in self.data} - - return ({".": importer} if importer else {}) - - def merge(self, lf): - """ - Merges two lockfiles: - 1. Converts the lockfile to monorepo-like lockfile with "importers" section instead of "dependencies" and "specifiers". - 2. Merges `lf`'s dependencies and specifiers to importers. - 3. Merges `lf`'s packages to the lockfile. - :param lf: lockfile to merge - :type lf: PnpmLockfile - """ - importers = self.get_importers() - build_path = os.path.dirname(self.path) - - for [importer, imports] in iteritems(lf.get_importers()): - importer_path = os.path.normpath(os.path.join(os.path.dirname(lf.path), importer)) - importer_rel_path = os.path.relpath(importer_path, build_path) - importers[importer_rel_path] = imports - - self.data["importers"] = importers - - for k in self.IMPORTER_KEYS: - self.data.pop(k, None) - - packages = self.data.get("packages", {}) - for k, v in iteritems(lf.data.get("packages", {})): - if k not in packages: - packages[k] = v - self.data["packages"] = packages - - -def _parse_package_meta(key, meta): - """ - :param key: uniq package key from lockfile - :type key: string - :param meta: package meta dict from lockfile - :type meta: dict - :rtype: LockfilePackageMetaInvalidError - """ - try: - name, version = _parse_package_key(key) - sky_id = _parse_sky_id_from_tarball_url(meta["resolution"]["tarball"]) - integrity_algorithm, integrity = _parse_package_integrity(meta["resolution"]["integrity"]) - except KeyError as e: - raise TypeError("Invalid package meta for key {}, missing {} key".format(key, e)) - except LockfilePackageMetaInvalidError as e: - raise TypeError("Invalid package meta for key {}, parse error: {}".format(key, e)) - - return LockfilePackageMeta(name, version, sky_id, integrity, integrity_algorithm) - - -def _parse_package_key(key): - """ - :param key: package key in format "/({scope}/)?{package_name}/{package_version}(_{peer_dependencies})?" - :type key: string - :return: tuple of scoped package name and version - :rtype: (str, str) - """ - try: - tokens = key.split("/")[1:] - version = tokens.pop().split("_", 1)[0] - - if len(tokens) < 1 or len(tokens) > 2: - raise TypeError() - except (IndexError, TypeError): - raise LockfilePackageMetaInvalidError("Invalid package key") - - return ("/".join(tokens), version) - - -def _parse_sky_id_from_tarball_url(tarball_url): - """ - :param tarball_url: tarball url - :type tarball_url: string - :return: sky id - :rtype: string - """ - if tarball_url.startswith("file:"): - return "" - - rbtorrent_param = urlparse.parse_qs(urlparse.urlparse(tarball_url).query).get("rbtorrent") - - if rbtorrent_param is None: - raise LockfilePackageMetaInvalidError("Missing rbtorrent param in tarball url {}".format(tarball_url)) - - return "rbtorrent:{}".format(rbtorrent_param[0]) - - -def _parse_package_integrity(integrity): - """ - :param integrity: package integrity in format "{algo}-{base64_of_hash}" - :type integrity: string - :return: tuple of algorithm and hash (hex) - :rtype: (str, str) - """ - algo, hash_b64 = integrity.split("-", 1) - - try: - hash_hex = binascii.hexlify(base64.b64decode(hash_b64)) - except TypeError as e: - raise LockfilePackageMetaInvalidError("Invalid package integrity encoding, integrity: {}, error: {}".format(integrity, e)) - - return (algo, hash_hex) +import base64 +import binascii +import yaml +import os + +from six.moves.urllib import parse as urlparse +from six import iteritems + +from ..base import PackageJson, BaseLockfile, LockfilePackageMeta, LockfilePackageMetaInvalidError + + +class PnpmLockfile(BaseLockfile): + IMPORTER_KEYS = PackageJson.DEP_KEYS + ("specifiers",) + + def read(self): + with open(self.path, "r") as f: + self.data = yaml.load(f, Loader=yaml.CSafeLoader) + + def write(self, path=None): + """ + :param path: path to store lockfile, defaults to original path + :type path: str + """ + if path is None: + path = self.path + + with open(path, "w") as f: + yaml.dump(self.data, f, Dumper=yaml.CSafeDumper) + + def get_packages_meta(self): + """ + Extracts packages meta from lockfile. + :rtype: list of LockfilePackageMeta + """ + packages = self.data.get("packages", {}) + + return map(lambda x: _parse_package_meta(*x), iteritems(packages)) + + def update_tarball_resolutions(self, fn): + """ + :param fn: maps `LockfilePackageMeta` instance to new `resolution.tarball` value + :type fn: lambda + """ + packages = self.data.get("packages", {}) + + for key, meta in iteritems(packages): + meta["resolution"]["tarball"] = fn(_parse_package_meta(key, meta)) + packages[key] = meta + + def get_importers(self): + """ + Returns "importers" section from the lockfile or creates similar structure from "dependencies" and "specifiers". + :rtype: dict of dict of dict of str + """ + importers = self.data.get("importers") + if importers is not None: + return importers + + importer = {k: self.data[k] for k in self.IMPORTER_KEYS if k in self.data} + + return ({".": importer} if importer else {}) + + def merge(self, lf): + """ + Merges two lockfiles: + 1. Converts the lockfile to monorepo-like lockfile with "importers" section instead of "dependencies" and "specifiers". + 2. Merges `lf`'s dependencies and specifiers to importers. + 3. Merges `lf`'s packages to the lockfile. + :param lf: lockfile to merge + :type lf: PnpmLockfile + """ + importers = self.get_importers() + build_path = os.path.dirname(self.path) + + for [importer, imports] in iteritems(lf.get_importers()): + importer_path = os.path.normpath(os.path.join(os.path.dirname(lf.path), importer)) + importer_rel_path = os.path.relpath(importer_path, build_path) + importers[importer_rel_path] = imports + + self.data["importers"] = importers + + for k in self.IMPORTER_KEYS: + self.data.pop(k, None) + + packages = self.data.get("packages", {}) + for k, v in iteritems(lf.data.get("packages", {})): + if k not in packages: + packages[k] = v + self.data["packages"] = packages + + +def _parse_package_meta(key, meta): + """ + :param key: uniq package key from lockfile + :type key: string + :param meta: package meta dict from lockfile + :type meta: dict + :rtype: LockfilePackageMetaInvalidError + """ + try: + name, version = _parse_package_key(key) + sky_id = _parse_sky_id_from_tarball_url(meta["resolution"]["tarball"]) + integrity_algorithm, integrity = _parse_package_integrity(meta["resolution"]["integrity"]) + except KeyError as e: + raise TypeError("Invalid package meta for key {}, missing {} key".format(key, e)) + except LockfilePackageMetaInvalidError as e: + raise TypeError("Invalid package meta for key {}, parse error: {}".format(key, e)) + + return LockfilePackageMeta(name, version, sky_id, integrity, integrity_algorithm) + + +def _parse_package_key(key): + """ + :param key: package key in format "/({scope}/)?{package_name}/{package_version}(_{peer_dependencies})?" + :type key: string + :return: tuple of scoped package name and version + :rtype: (str, str) + """ + try: + tokens = key.split("/")[1:] + version = tokens.pop().split("_", 1)[0] + + if len(tokens) < 1 or len(tokens) > 2: + raise TypeError() + except (IndexError, TypeError): + raise LockfilePackageMetaInvalidError("Invalid package key") + + return ("/".join(tokens), version) + + +def _parse_sky_id_from_tarball_url(tarball_url): + """ + :param tarball_url: tarball url + :type tarball_url: string + :return: sky id + :rtype: string + """ + if tarball_url.startswith("file:"): + return "" + + rbtorrent_param = urlparse.parse_qs(urlparse.urlparse(tarball_url).query).get("rbtorrent") + + if rbtorrent_param is None: + raise LockfilePackageMetaInvalidError("Missing rbtorrent param in tarball url {}".format(tarball_url)) + + return "rbtorrent:{}".format(rbtorrent_param[0]) + + +def _parse_package_integrity(integrity): + """ + :param integrity: package integrity in format "{algo}-{base64_of_hash}" + :type integrity: string + :return: tuple of algorithm and hash (hex) + :rtype: (str, str) + """ + algo, hash_b64 = integrity.split("-", 1) + + try: + hash_hex = binascii.hexlify(base64.b64decode(hash_b64)) + except TypeError as e: + raise LockfilePackageMetaInvalidError("Invalid package integrity encoding, integrity: {}, error: {}".format(integrity, e)) + + return (algo, hash_hex) diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py index 81692c825f..1a48675834 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py +++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py @@ -1,181 +1,181 @@ -import os -import shutil -import yaml - -from six import iteritems - -from ..base import PackageJson, BasePackageManager, PackageManagerError -from .lockfile import PnpmLockfile -from .workspace import PnpmWorkspace -from .utils import build_pj_path, build_lockfile_path, build_ws_config_path, build_nm_bundle_path - - -class PnpmPackageManager(BasePackageManager): - _STORE_NM_PATH = os.path.join(".pnpm", "store") - _VSTORE_NM_PATH = os.path.join(".pnpm", "virtual-store") - _STORE_VER = "v3" - - def install(self): - """ - Creates node_modules directory according to the lockfile. - """ - self._prepare_workspace() - self._exec_command([ - "install", - "--offline", - "--frozen-lockfile", - "--store-dir", self._nm_path(self._STORE_NM_PATH), - "--virtual-store-dir", self._nm_path(self._VSTORE_NM_PATH), - "--no-verify-store-integrity", - "--package-import-method", "hardlink", - "--ignore-pnpmfile", - "--ignore-scripts", - "--strict-peer-dependencies", - ]) - self._fix_stores_in_modules_yaml() - - def get_peer_paths_from_package_json(self): - """ - Returns paths of direct workspace dependencies (source root related). - :rtype: list of str - """ - pj = PackageJson.load(build_pj_path(self.sources_path)) - - return map(lambda x: os.path.normpath(os.path.join(self.module_path, x[1])), pj.get_workspace_dep_paths()) - - def calc_node_modules_inouts(self): - """ - Returns input and output paths for command that creates `node_modules` bundle. - :return: Pair of input and output paths with correct roots ($S or $B). - :rtype: (list of str, list of str) - """ - # Inputs: source package.json and lockfile, built package.jsons, lockfiles and workspace configs of deps, tarballs. - ins = [] - # Source lockfiles are used only to get tarballs info. - src_lf_paths = [build_lockfile_path(self.sources_path)] - pj = PackageJson.load(build_pj_path(self.sources_path)) - - for [dep_src_path, (dep_pj, depth)] in iteritems(pj.get_workspace_map()): - if dep_src_path == self.sources_path: - continue - dep_mod_path = dep_src_path[len(self.sources_root) + 1:] - # pnpm requires all package.jsons. - ins.append(build_pj_path(dep_mod_path)) - dep_lf_src_path = build_lockfile_path(dep_src_path) - if not os.path.isfile(dep_lf_src_path): - continue - src_lf_paths.append(dep_lf_src_path) - # Merged workspace configs and lockfiles of direct deps. - if depth == 1: - ins.append(build_ws_config_path(dep_mod_path)) - ins.append(build_lockfile_path(dep_mod_path)) - - for pkg in self.extract_packages_meta_from_lockfiles(src_lf_paths): - ins.append(self._contrib_tarball_path(pkg)) - - s_root = lambda x: os.path.join("$S", x) - b_root = lambda x: os.path.join("$B", x) - - ins = map(b_root, ins) + [ - s_root(build_pj_path(self.module_path)), - s_root(build_lockfile_path(self.module_path)), - ] - - # Outputs: patched lockfile, generated workspace config, created node_modules bundle. - outs = [b_root(f(self.module_path)) for f in (build_lockfile_path, build_ws_config_path, build_nm_bundle_path)] - - return (ins, outs) - - def extract_packages_meta_from_lockfiles(self, lf_paths): - """ - :type lf_paths: iterable of BaseLockfile - :rtype: iterable of LockfilePackageMeta - """ - tarballs = set() - - for lf_path in lf_paths: - try: - for pkg in PnpmLockfile.load(lf_path).get_packages_meta(): - if pkg.tarball_path not in tarballs: - tarballs.add(pkg.tarball_path) - yield pkg - except Exception as e: - raise PackageManagerError("Unable to process lockfile {}: {}".format(lf_path, e)) - - def _prepare_workspace(self): - pj = self._build_package_json() - ws = PnpmWorkspace(build_ws_config_path(self.build_path)) - ws.set_from_package_json(pj) - dep_paths = ws.get_paths() - self._build_merged_workspace_config(ws, dep_paths) - self._build_merged_lockfile(dep_paths) - - def _build_package_json(self): - """ - :rtype: PackageJson - """ - in_pj_path = build_pj_path(self.sources_path) - out_pj_path = build_pj_path(self.build_path) - shutil.copyfile(in_pj_path, out_pj_path) - - return PackageJson.load(out_pj_path) - - def _build_merged_lockfile(self, dep_paths): - """ - :type dep_paths: list of str - :rtype: PnpmLockfile - """ - in_lf_path = build_lockfile_path(self.sources_path) - out_lf_path = build_lockfile_path(self.build_path) - - lf = PnpmLockfile.load(in_lf_path) - # Change to the output path for correct path calcs on merging. - lf.path = out_lf_path - - for dep_path in dep_paths: - if dep_path is self.build_path: - continue - lf_path = build_lockfile_path(dep_path) - if os.path.isfile(lf_path): - lf.merge(PnpmLockfile.load(lf_path)) - - lf.update_tarball_resolutions(lambda p: self._contrib_tarball_url(p)) - lf.write() - - def _build_merged_workspace_config(self, ws, dep_paths): - """ - :type ws: PnpmWorkspaceConfig - :type dep_paths: list of str - """ - for dep_path in dep_paths: - if dep_path is self.build_path: - continue - ws_config_path = build_ws_config_path(dep_path) - if os.path.isfile(ws_config_path): - ws.merge(PnpmWorkspace.load(ws_config_path)) - - ws.write() - - def _fix_stores_in_modules_yaml(self): - """ - Ensures that store paths are the same as would be after installing deps in the source dir. - This is required to reuse `node_modules` after build. - """ - with open(self._nm_path(".modules.yaml"), "r+") as f: - data = yaml.load(f, Loader=yaml.CSafeLoader) - # NOTE: pnpm requires absolute store path here. - data["storeDir"] = os.path.join(self.sources_path, "node_modules", self._STORE_NM_PATH, self._STORE_VER) - data["virtualStoreDir"] = self._VSTORE_NM_PATH - f.seek(0) - yaml.dump(data, f, Dumper=yaml.CSafeDumper) - f.truncate() - - def _get_default_options(self): - return super(PnpmPackageManager, self)._get_default_options() + [ - "--stream", - "--reporter", "append-only", - "--no-color", - ] - - def _get_debug_log_path(self): - return self._nm_path(".pnpm-debug.log") +import os +import shutil +import yaml + +from six import iteritems + +from ..base import PackageJson, BasePackageManager, PackageManagerError +from .lockfile import PnpmLockfile +from .workspace import PnpmWorkspace +from .utils import build_pj_path, build_lockfile_path, build_ws_config_path, build_nm_bundle_path + + +class PnpmPackageManager(BasePackageManager): + _STORE_NM_PATH = os.path.join(".pnpm", "store") + _VSTORE_NM_PATH = os.path.join(".pnpm", "virtual-store") + _STORE_VER = "v3" + + def install(self): + """ + Creates node_modules directory according to the lockfile. + """ + self._prepare_workspace() + self._exec_command([ + "install", + "--offline", + "--frozen-lockfile", + "--store-dir", self._nm_path(self._STORE_NM_PATH), + "--virtual-store-dir", self._nm_path(self._VSTORE_NM_PATH), + "--no-verify-store-integrity", + "--package-import-method", "hardlink", + "--ignore-pnpmfile", + "--ignore-scripts", + "--strict-peer-dependencies", + ]) + self._fix_stores_in_modules_yaml() + + def get_peer_paths_from_package_json(self): + """ + Returns paths of direct workspace dependencies (source root related). + :rtype: list of str + """ + pj = PackageJson.load(build_pj_path(self.sources_path)) + + return map(lambda x: os.path.normpath(os.path.join(self.module_path, x[1])), pj.get_workspace_dep_paths()) + + def calc_node_modules_inouts(self): + """ + Returns input and output paths for command that creates `node_modules` bundle. + :return: Pair of input and output paths with correct roots ($S or $B). + :rtype: (list of str, list of str) + """ + # Inputs: source package.json and lockfile, built package.jsons, lockfiles and workspace configs of deps, tarballs. + ins = [] + # Source lockfiles are used only to get tarballs info. + src_lf_paths = [build_lockfile_path(self.sources_path)] + pj = PackageJson.load(build_pj_path(self.sources_path)) + + for [dep_src_path, (dep_pj, depth)] in iteritems(pj.get_workspace_map()): + if dep_src_path == self.sources_path: + continue + dep_mod_path = dep_src_path[len(self.sources_root) + 1:] + # pnpm requires all package.jsons. + ins.append(build_pj_path(dep_mod_path)) + dep_lf_src_path = build_lockfile_path(dep_src_path) + if not os.path.isfile(dep_lf_src_path): + continue + src_lf_paths.append(dep_lf_src_path) + # Merged workspace configs and lockfiles of direct deps. + if depth == 1: + ins.append(build_ws_config_path(dep_mod_path)) + ins.append(build_lockfile_path(dep_mod_path)) + + for pkg in self.extract_packages_meta_from_lockfiles(src_lf_paths): + ins.append(self._contrib_tarball_path(pkg)) + + s_root = lambda x: os.path.join("$S", x) + b_root = lambda x: os.path.join("$B", x) + + ins = map(b_root, ins) + [ + s_root(build_pj_path(self.module_path)), + s_root(build_lockfile_path(self.module_path)), + ] + + # Outputs: patched lockfile, generated workspace config, created node_modules bundle. + outs = [b_root(f(self.module_path)) for f in (build_lockfile_path, build_ws_config_path, build_nm_bundle_path)] + + return (ins, outs) + + def extract_packages_meta_from_lockfiles(self, lf_paths): + """ + :type lf_paths: iterable of BaseLockfile + :rtype: iterable of LockfilePackageMeta + """ + tarballs = set() + + for lf_path in lf_paths: + try: + for pkg in PnpmLockfile.load(lf_path).get_packages_meta(): + if pkg.tarball_path not in tarballs: + tarballs.add(pkg.tarball_path) + yield pkg + except Exception as e: + raise PackageManagerError("Unable to process lockfile {}: {}".format(lf_path, e)) + + def _prepare_workspace(self): + pj = self._build_package_json() + ws = PnpmWorkspace(build_ws_config_path(self.build_path)) + ws.set_from_package_json(pj) + dep_paths = ws.get_paths() + self._build_merged_workspace_config(ws, dep_paths) + self._build_merged_lockfile(dep_paths) + + def _build_package_json(self): + """ + :rtype: PackageJson + """ + in_pj_path = build_pj_path(self.sources_path) + out_pj_path = build_pj_path(self.build_path) + shutil.copyfile(in_pj_path, out_pj_path) + + return PackageJson.load(out_pj_path) + + def _build_merged_lockfile(self, dep_paths): + """ + :type dep_paths: list of str + :rtype: PnpmLockfile + """ + in_lf_path = build_lockfile_path(self.sources_path) + out_lf_path = build_lockfile_path(self.build_path) + + lf = PnpmLockfile.load(in_lf_path) + # Change to the output path for correct path calcs on merging. + lf.path = out_lf_path + + for dep_path in dep_paths: + if dep_path is self.build_path: + continue + lf_path = build_lockfile_path(dep_path) + if os.path.isfile(lf_path): + lf.merge(PnpmLockfile.load(lf_path)) + + lf.update_tarball_resolutions(lambda p: self._contrib_tarball_url(p)) + lf.write() + + def _build_merged_workspace_config(self, ws, dep_paths): + """ + :type ws: PnpmWorkspaceConfig + :type dep_paths: list of str + """ + for dep_path in dep_paths: + if dep_path is self.build_path: + continue + ws_config_path = build_ws_config_path(dep_path) + if os.path.isfile(ws_config_path): + ws.merge(PnpmWorkspace.load(ws_config_path)) + + ws.write() + + def _fix_stores_in_modules_yaml(self): + """ + Ensures that store paths are the same as would be after installing deps in the source dir. + This is required to reuse `node_modules` after build. + """ + with open(self._nm_path(".modules.yaml"), "r+") as f: + data = yaml.load(f, Loader=yaml.CSafeLoader) + # NOTE: pnpm requires absolute store path here. + data["storeDir"] = os.path.join(self.sources_path, "node_modules", self._STORE_NM_PATH, self._STORE_VER) + data["virtualStoreDir"] = self._VSTORE_NM_PATH + f.seek(0) + yaml.dump(data, f, Dumper=yaml.CSafeDumper) + f.truncate() + + def _get_default_options(self): + return super(PnpmPackageManager, self)._get_default_options() + [ + "--stream", + "--reporter", "append-only", + "--no-color", + ] + + def _get_debug_log_path(self): + return self._nm_path(".pnpm-debug.log") diff --git a/build/plugins/lib/nots/package_manager/pnpm/tests/lockfile.py b/build/plugins/lib/nots/package_manager/pnpm/tests/lockfile.py index a4e4b68eb7..06315a4992 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/tests/lockfile.py +++ b/build/plugins/lib/nots/package_manager/pnpm/tests/lockfile.py @@ -1,320 +1,320 @@ -import pytest - -from build.plugins.lib.nots.package_manager.pnpm.lockfile import PnpmLockfile - - -def test_lockfile_get_packages_meta_ok(): - lf = PnpmLockfile(path="/pnpm-lock.yaml") - lf.data = { - "packages": { - "/@babel/cli/7.6.2_@babel+core@7.6.2": { - "resolution": { - "integrity": "sha512-JDZ+T/br9pPfT2lmAMJypJDTTTHM9ePD/ED10TRjRzJVdEVy+JB3iRlhzYmTt5YkNgHvxWGlUVnLtdv6ruiDrQ==", - "tarball": "@babel%2fcli/-/cli-7.6.2.tgz?rbtorrent=cb1849da3e4947e56a8f6bde6a1ec42703ddd187", - }, - }, - }, - } - - packages = list(lf.get_packages_meta()) - pkg = packages[0] - - assert len(packages) == 1 - assert pkg.name == "@babel/cli" - assert pkg.version == "7.6.2" - assert pkg.sky_id == "rbtorrent:cb1849da3e4947e56a8f6bde6a1ec42703ddd187" - assert pkg.integrity == b"24367e4ff6ebf693df4f696600c272a490d34d31ccf5e3c3fc40f5d13463473255744572f89077891961cd8993b796243601efc561a55159cbb5dbfaaee883ad" - assert pkg.integrity_algorithm == "sha512" - - -def test_lockfile_get_packages_empty(): - lf = PnpmLockfile(path="/pnpm-lock.yaml") - lf.data = {} - - assert len(list(lf.get_packages_meta())) == 0 - - -def test_package_meta_invalid_key(): - lf = PnpmLockfile(path="/pnpm-lock.yaml") - lf.data = { - "packages": { - "in/valid": {}, - }, - } - - with pytest.raises(TypeError) as e: - list(lf.get_packages_meta()) - - assert str(e.value) == "Invalid package meta for key in/valid, parse error: Invalid package key" - - -def test_package_meta_missing_resolution(): - lf = PnpmLockfile(path="/pnpm-lock.yaml") - lf.data = { - "packages": { - "/valid/1.2.3": {}, - }, - } - - with pytest.raises(TypeError) as e: - list(lf.get_packages_meta()) - - assert str(e.value) == "Invalid package meta for key /valid/1.2.3, missing 'resolution' key" - - -def test_package_meta_missing_tarball(): - lf = PnpmLockfile(path="/pnpm-lock.yaml") - lf.data = { - "packages": { - "/valid/1.2.3": { - "resolution": {}, - }, - }, - } - - with pytest.raises(TypeError) as e: - list(lf.get_packages_meta()) - - assert str(e.value) == "Invalid package meta for key /valid/1.2.3, missing 'tarball' key" - - -def test_package_meta_missing_rbtorrent(): - lf = PnpmLockfile(path="/pnpm-lock.yaml") - lf.data = { - "packages": { - "/valid/1.2.3": { - "resolution": { - "tarball": "valid-1.2.3.tgz", - }, - }, - }, - } - - with pytest.raises(TypeError) as e: - list(lf.get_packages_meta()) - - assert str(e.value) == "Invalid package meta for key /valid/1.2.3, parse error: Missing rbtorrent param in tarball url valid-1.2.3.tgz" - - -def test_lockfile_meta_file_tarball(): - lf = PnpmLockfile(path="/pnpm-lock.yaml") - lf.data = { - "packages": { - "/@babel/cli/7.6.2": { - "resolution": { - "integrity": "sha512-JDZ+T/br9pPfT2lmAMJypJDTTTHM9ePD/ED10TRjRzJVdEVy+JB3iRlhzYmTt5YkNgHvxWGlUVnLtdv6ruiDrQ==", - "tarball": "file:/some/abs/path.tgz", - }, - }, - }, - } - - packages = list(lf.get_packages_meta()) - pkg = packages[0] - - assert len(packages) == 1 - assert pkg.name == "@babel/cli" - assert pkg.version == "7.6.2" - assert pkg.sky_id == "" - - -def test_lockfile_update_tarball_resolutions_ok(): - lf = PnpmLockfile(path="/pnpm-lock.yaml") - lf.data = { - "packages": { - "/@babel/cli/7.6.2_@babel+core@7.6.2": { - "resolution": { - "integrity": "sha512-JDZ+T/br9pPfT2lmAMJypJDTTTHM9ePD/ED10TRjRzJVdEVy+JB3iRlhzYmTt5YkNgHvxWGlUVnLtdv6ruiDrQ==", - "tarball": "@babel%2fcli/-/cli-7.6.2.tgz?rbtorrent=cb1849da3e4947e56a8f6bde6a1ec42703ddd187", - }, - }, - }, - } - - lf.update_tarball_resolutions(lambda p: p.name) - - assert lf.data["packages"]["/@babel/cli/7.6.2_@babel+core@7.6.2"]["resolution"]["tarball"] == "@babel/cli" - - -def test_lockfile_merge(): - lf1 = PnpmLockfile(path="/foo/pnpm-lock.yaml") - lf1.data = { - "dependencies": { - "a": "1.0.0", - }, - "specifiers": { - "a": "1.0.0", - }, - "packages": { - "/a/1.0.0": {}, - }, - } - - lf2 = PnpmLockfile(path="/bar/pnpm-lock.yaml") - lf2.data = { - "dependencies": { - "b": "1.0.0", - }, - "specifiers": { - "b": "1.0.0", - }, - "packages": { - "/b/1.0.0": {}, - }, - } - - lf3 = PnpmLockfile(path="/another/baz/pnpm-lock.yaml") - lf3.data = { - "importers": { - ".": { - "dependencies": { - "@a/qux": "link:../qux", - "a": "1.0.0", - }, - "specifiers": { - "@a/qux": "workspace:../qux", - "a": "1.0.0", - }, - }, - "../qux": { - "dependencies": { - "b": "1.0.1", - }, - "specifiers": { - "b": "1.0.1", - }, - }, - }, - "packages": { - "/a/1.0.0": {}, - "/b/1.0.1": {}, - }, - } - - lf4 = PnpmLockfile(path="/another/quux/pnpm-lock.yaml") - lf4.data = { - "dependencies": { - "@a/bar": "link:../../bar", - }, - "specifiers": { - "@a/bar": "workspace:../../bar", - }, - } - - lf1.merge(lf2) - lf1.merge(lf3) - lf1.merge(lf4) - - assert lf1.data == { - "importers": { - ".": { - "dependencies": { - "a": "1.0.0", - }, - "specifiers": { - "a": "1.0.0", - }, - }, - "../bar": { - "dependencies": { - "b": "1.0.0", - }, - "specifiers": { - "b": "1.0.0", - }, - }, - "../another/baz": { - "dependencies": { - "@a/qux": "link:../qux", - "a": "1.0.0", - }, - "specifiers": { - "@a/qux": "workspace:../qux", - "a": "1.0.0", - }, - }, - "../another/qux": { - "dependencies": { - "b": "1.0.1", - }, - "specifiers": { - "b": "1.0.1", - }, - }, - "../another/quux": { - "dependencies": { - "@a/bar": "link:../../bar", - }, - "specifiers": { - "@a/bar": "workspace:../../bar", - }, - }, - }, - "packages": { - "/a/1.0.0": {}, - "/b/1.0.0": {}, - "/b/1.0.1": {}, - }, - } - - -def test_lockfile_merge_dont_overrides_packages(): - lf1 = PnpmLockfile(path="/foo/pnpm-lock.yaml") - lf1.data = { - "dependencies": { - "a": "1.0.0", - }, - "specifiers": { - "a": "1.0.0", - }, - "packages": { - "/a/1.0.0": {}, - }, - } - - lf2 = PnpmLockfile(path="/bar/pnpm-lock.yaml") - lf2.data = { - "dependencies": { - "a": "1.0.0", - "b": "1.0.0", - }, - "specifiers": { - "a": "1.0.0", - "b": "1.0.0", - }, - "packages": { - "/a/1.0.0": { - "overriden": True, - }, - "/b/1.0.0": {}, - }, - } - - lf1.merge(lf2) - - assert lf1.data == { - "importers": { - ".": { - "dependencies": { - "a": "1.0.0", - }, - "specifiers": { - "a": "1.0.0", - }, - }, - "../bar": { - "dependencies": { - "a": "1.0.0", - "b": "1.0.0", - }, - "specifiers": { - "a": "1.0.0", - "b": "1.0.0", - }, - }, - }, - "packages": { - "/a/1.0.0": {}, - "/b/1.0.0": {}, - }, - } +import pytest + +from build.plugins.lib.nots.package_manager.pnpm.lockfile import PnpmLockfile + + +def test_lockfile_get_packages_meta_ok(): + lf = PnpmLockfile(path="/pnpm-lock.yaml") + lf.data = { + "packages": { + "/@babel/cli/7.6.2_@babel+core@7.6.2": { + "resolution": { + "integrity": "sha512-JDZ+T/br9pPfT2lmAMJypJDTTTHM9ePD/ED10TRjRzJVdEVy+JB3iRlhzYmTt5YkNgHvxWGlUVnLtdv6ruiDrQ==", + "tarball": "@babel%2fcli/-/cli-7.6.2.tgz?rbtorrent=cb1849da3e4947e56a8f6bde6a1ec42703ddd187", + }, + }, + }, + } + + packages = list(lf.get_packages_meta()) + pkg = packages[0] + + assert len(packages) == 1 + assert pkg.name == "@babel/cli" + assert pkg.version == "7.6.2" + assert pkg.sky_id == "rbtorrent:cb1849da3e4947e56a8f6bde6a1ec42703ddd187" + assert pkg.integrity == b"24367e4ff6ebf693df4f696600c272a490d34d31ccf5e3c3fc40f5d13463473255744572f89077891961cd8993b796243601efc561a55159cbb5dbfaaee883ad" + assert pkg.integrity_algorithm == "sha512" + + +def test_lockfile_get_packages_empty(): + lf = PnpmLockfile(path="/pnpm-lock.yaml") + lf.data = {} + + assert len(list(lf.get_packages_meta())) == 0 + + +def test_package_meta_invalid_key(): + lf = PnpmLockfile(path="/pnpm-lock.yaml") + lf.data = { + "packages": { + "in/valid": {}, + }, + } + + with pytest.raises(TypeError) as e: + list(lf.get_packages_meta()) + + assert str(e.value) == "Invalid package meta for key in/valid, parse error: Invalid package key" + + +def test_package_meta_missing_resolution(): + lf = PnpmLockfile(path="/pnpm-lock.yaml") + lf.data = { + "packages": { + "/valid/1.2.3": {}, + }, + } + + with pytest.raises(TypeError) as e: + list(lf.get_packages_meta()) + + assert str(e.value) == "Invalid package meta for key /valid/1.2.3, missing 'resolution' key" + + +def test_package_meta_missing_tarball(): + lf = PnpmLockfile(path="/pnpm-lock.yaml") + lf.data = { + "packages": { + "/valid/1.2.3": { + "resolution": {}, + }, + }, + } + + with pytest.raises(TypeError) as e: + list(lf.get_packages_meta()) + + assert str(e.value) == "Invalid package meta for key /valid/1.2.3, missing 'tarball' key" + + +def test_package_meta_missing_rbtorrent(): + lf = PnpmLockfile(path="/pnpm-lock.yaml") + lf.data = { + "packages": { + "/valid/1.2.3": { + "resolution": { + "tarball": "valid-1.2.3.tgz", + }, + }, + }, + } + + with pytest.raises(TypeError) as e: + list(lf.get_packages_meta()) + + assert str(e.value) == "Invalid package meta for key /valid/1.2.3, parse error: Missing rbtorrent param in tarball url valid-1.2.3.tgz" + + +def test_lockfile_meta_file_tarball(): + lf = PnpmLockfile(path="/pnpm-lock.yaml") + lf.data = { + "packages": { + "/@babel/cli/7.6.2": { + "resolution": { + "integrity": "sha512-JDZ+T/br9pPfT2lmAMJypJDTTTHM9ePD/ED10TRjRzJVdEVy+JB3iRlhzYmTt5YkNgHvxWGlUVnLtdv6ruiDrQ==", + "tarball": "file:/some/abs/path.tgz", + }, + }, + }, + } + + packages = list(lf.get_packages_meta()) + pkg = packages[0] + + assert len(packages) == 1 + assert pkg.name == "@babel/cli" + assert pkg.version == "7.6.2" + assert pkg.sky_id == "" + + +def test_lockfile_update_tarball_resolutions_ok(): + lf = PnpmLockfile(path="/pnpm-lock.yaml") + lf.data = { + "packages": { + "/@babel/cli/7.6.2_@babel+core@7.6.2": { + "resolution": { + "integrity": "sha512-JDZ+T/br9pPfT2lmAMJypJDTTTHM9ePD/ED10TRjRzJVdEVy+JB3iRlhzYmTt5YkNgHvxWGlUVnLtdv6ruiDrQ==", + "tarball": "@babel%2fcli/-/cli-7.6.2.tgz?rbtorrent=cb1849da3e4947e56a8f6bde6a1ec42703ddd187", + }, + }, + }, + } + + lf.update_tarball_resolutions(lambda p: p.name) + + assert lf.data["packages"]["/@babel/cli/7.6.2_@babel+core@7.6.2"]["resolution"]["tarball"] == "@babel/cli" + + +def test_lockfile_merge(): + lf1 = PnpmLockfile(path="/foo/pnpm-lock.yaml") + lf1.data = { + "dependencies": { + "a": "1.0.0", + }, + "specifiers": { + "a": "1.0.0", + }, + "packages": { + "/a/1.0.0": {}, + }, + } + + lf2 = PnpmLockfile(path="/bar/pnpm-lock.yaml") + lf2.data = { + "dependencies": { + "b": "1.0.0", + }, + "specifiers": { + "b": "1.0.0", + }, + "packages": { + "/b/1.0.0": {}, + }, + } + + lf3 = PnpmLockfile(path="/another/baz/pnpm-lock.yaml") + lf3.data = { + "importers": { + ".": { + "dependencies": { + "@a/qux": "link:../qux", + "a": "1.0.0", + }, + "specifiers": { + "@a/qux": "workspace:../qux", + "a": "1.0.0", + }, + }, + "../qux": { + "dependencies": { + "b": "1.0.1", + }, + "specifiers": { + "b": "1.0.1", + }, + }, + }, + "packages": { + "/a/1.0.0": {}, + "/b/1.0.1": {}, + }, + } + + lf4 = PnpmLockfile(path="/another/quux/pnpm-lock.yaml") + lf4.data = { + "dependencies": { + "@a/bar": "link:../../bar", + }, + "specifiers": { + "@a/bar": "workspace:../../bar", + }, + } + + lf1.merge(lf2) + lf1.merge(lf3) + lf1.merge(lf4) + + assert lf1.data == { + "importers": { + ".": { + "dependencies": { + "a": "1.0.0", + }, + "specifiers": { + "a": "1.0.0", + }, + }, + "../bar": { + "dependencies": { + "b": "1.0.0", + }, + "specifiers": { + "b": "1.0.0", + }, + }, + "../another/baz": { + "dependencies": { + "@a/qux": "link:../qux", + "a": "1.0.0", + }, + "specifiers": { + "@a/qux": "workspace:../qux", + "a": "1.0.0", + }, + }, + "../another/qux": { + "dependencies": { + "b": "1.0.1", + }, + "specifiers": { + "b": "1.0.1", + }, + }, + "../another/quux": { + "dependencies": { + "@a/bar": "link:../../bar", + }, + "specifiers": { + "@a/bar": "workspace:../../bar", + }, + }, + }, + "packages": { + "/a/1.0.0": {}, + "/b/1.0.0": {}, + "/b/1.0.1": {}, + }, + } + + +def test_lockfile_merge_dont_overrides_packages(): + lf1 = PnpmLockfile(path="/foo/pnpm-lock.yaml") + lf1.data = { + "dependencies": { + "a": "1.0.0", + }, + "specifiers": { + "a": "1.0.0", + }, + "packages": { + "/a/1.0.0": {}, + }, + } + + lf2 = PnpmLockfile(path="/bar/pnpm-lock.yaml") + lf2.data = { + "dependencies": { + "a": "1.0.0", + "b": "1.0.0", + }, + "specifiers": { + "a": "1.0.0", + "b": "1.0.0", + }, + "packages": { + "/a/1.0.0": { + "overriden": True, + }, + "/b/1.0.0": {}, + }, + } + + lf1.merge(lf2) + + assert lf1.data == { + "importers": { + ".": { + "dependencies": { + "a": "1.0.0", + }, + "specifiers": { + "a": "1.0.0", + }, + }, + "../bar": { + "dependencies": { + "a": "1.0.0", + "b": "1.0.0", + }, + "specifiers": { + "a": "1.0.0", + "b": "1.0.0", + }, + }, + }, + "packages": { + "/a/1.0.0": {}, + "/b/1.0.0": {}, + }, + } diff --git a/build/plugins/lib/nots/package_manager/pnpm/tests/workspace.py b/build/plugins/lib/nots/package_manager/pnpm/tests/workspace.py index 8a39710821..f6a73e0d4c 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/tests/workspace.py +++ b/build/plugins/lib/nots/package_manager/pnpm/tests/workspace.py @@ -1,58 +1,58 @@ -from build.plugins.lib.nots.package_manager.base import PackageJson -from build.plugins.lib.nots.package_manager.pnpm.workspace import PnpmWorkspace - - -def test_workspace_get_paths(): - ws = PnpmWorkspace(path="/packages/foo/pnpm-workspace.yaml") - ws.packages = set([".", "../bar", "../../another/baz"]) - - assert sorted(ws.get_paths()) == [ - "/another/baz", - "/packages/bar", - "/packages/foo", - ] - - -def test_workspace_set_from_package_json(): - ws = PnpmWorkspace(path="/packages/foo/pnpm-workspace.yaml") - pj = PackageJson(path="/packages/foo/package.json") - pj.data = { - "dependencies": { - "@a/bar": "workspace:../bar", - }, - "devDependencies": { - "@a/baz": "workspace:../../another/baz", - }, - "peerDependencies": { - "@a/qux": "workspace:../../another/qux", - }, - "optionalDependencies": { - "@a/quux": "workspace:../../another/quux", - } - } - - ws.set_from_package_json(pj) - - assert sorted(ws.get_paths()) == [ - "/another/baz", - "/another/quux", - "/another/qux", - "/packages/bar", - "/packages/foo", - ] - - -def test_workspace_merge(): - ws1 = PnpmWorkspace(path="/packages/foo/pnpm-workspace.yaml") - ws1.packages = set([".", "../bar", "../../another/baz"]) - ws2 = PnpmWorkspace(path="/another/baz/pnpm-workspace.yaml") - ws2.packages = set([".", "../qux"]) - - ws1.merge(ws2) - - assert sorted(ws1.get_paths()) == [ - "/another/baz", - "/another/qux", - "/packages/bar", - "/packages/foo", - ] +from build.plugins.lib.nots.package_manager.base import PackageJson +from build.plugins.lib.nots.package_manager.pnpm.workspace import PnpmWorkspace + + +def test_workspace_get_paths(): + ws = PnpmWorkspace(path="/packages/foo/pnpm-workspace.yaml") + ws.packages = set([".", "../bar", "../../another/baz"]) + + assert sorted(ws.get_paths()) == [ + "/another/baz", + "/packages/bar", + "/packages/foo", + ] + + +def test_workspace_set_from_package_json(): + ws = PnpmWorkspace(path="/packages/foo/pnpm-workspace.yaml") + pj = PackageJson(path="/packages/foo/package.json") + pj.data = { + "dependencies": { + "@a/bar": "workspace:../bar", + }, + "devDependencies": { + "@a/baz": "workspace:../../another/baz", + }, + "peerDependencies": { + "@a/qux": "workspace:../../another/qux", + }, + "optionalDependencies": { + "@a/quux": "workspace:../../another/quux", + } + } + + ws.set_from_package_json(pj) + + assert sorted(ws.get_paths()) == [ + "/another/baz", + "/another/quux", + "/another/qux", + "/packages/bar", + "/packages/foo", + ] + + +def test_workspace_merge(): + ws1 = PnpmWorkspace(path="/packages/foo/pnpm-workspace.yaml") + ws1.packages = set([".", "../bar", "../../another/baz"]) + ws2 = PnpmWorkspace(path="/another/baz/pnpm-workspace.yaml") + ws2.packages = set([".", "../qux"]) + + ws1.merge(ws2) + + assert sorted(ws1.get_paths()) == [ + "/another/baz", + "/another/qux", + "/packages/bar", + "/packages/foo", + ] diff --git a/build/plugins/lib/nots/package_manager/pnpm/tests/ya.make b/build/plugins/lib/nots/package_manager/pnpm/tests/ya.make index 1a0cc8d4fc..94712f1db9 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/tests/ya.make +++ b/build/plugins/lib/nots/package_manager/pnpm/tests/ya.make @@ -1,15 +1,15 @@ -PY23_TEST() - -OWNER(dankolesnikov) - -TEST_SRCS( - lockfile.py - workspace.py -) - -PEERDIR( - build/plugins/lib/nots/package_manager/base - build/plugins/lib/nots/package_manager/pnpm -) - -END() +PY23_TEST() + +OWNER(dankolesnikov) + +TEST_SRCS( + lockfile.py + workspace.py +) + +PEERDIR( + build/plugins/lib/nots/package_manager/base + build/plugins/lib/nots/package_manager/pnpm +) + +END() diff --git a/build/plugins/lib/nots/package_manager/pnpm/utils.py b/build/plugins/lib/nots/package_manager/pnpm/utils.py index 5cdec184d2..d8e99e3ab8 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/utils.py +++ b/build/plugins/lib/nots/package_manager/pnpm/utils.py @@ -1,19 +1,19 @@ -import os - -from ..base.constants import PACKAGE_JSON_FILENAME, PNPM_LOCKFILE_FILENAME, PNPM_WS_FILENAME, NODE_MODULES_BUNDLE_FILENAME - - -def build_pj_path(p): - return os.path.join(p, PACKAGE_JSON_FILENAME) - - -def build_lockfile_path(p): - return os.path.join(p, PNPM_LOCKFILE_FILENAME) - - -def build_ws_config_path(p): - return os.path.join(p, PNPM_WS_FILENAME) - - -def build_nm_bundle_path(p): - return os.path.join(p, NODE_MODULES_BUNDLE_FILENAME) +import os + +from ..base.constants import PACKAGE_JSON_FILENAME, PNPM_LOCKFILE_FILENAME, PNPM_WS_FILENAME, NODE_MODULES_BUNDLE_FILENAME + + +def build_pj_path(p): + return os.path.join(p, PACKAGE_JSON_FILENAME) + + +def build_lockfile_path(p): + return os.path.join(p, PNPM_LOCKFILE_FILENAME) + + +def build_ws_config_path(p): + return os.path.join(p, PNPM_WS_FILENAME) + + +def build_nm_bundle_path(p): + return os.path.join(p, NODE_MODULES_BUNDLE_FILENAME) diff --git a/build/plugins/lib/nots/package_manager/pnpm/workspace.py b/build/plugins/lib/nots/package_manager/pnpm/workspace.py index 8f1b8c8bbd..635b77dcb2 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/workspace.py +++ b/build/plugins/lib/nots/package_manager/pnpm/workspace.py @@ -1,69 +1,69 @@ -import os -import yaml - - -class PnpmWorkspace(object): - @classmethod - def load(cls, path): - ws = cls(path) - ws.read() - - return ws - - def __init__(self, path): - if not os.path.isabs(path): - raise TypeError("Absolute path required, given: {}".format(path)) - - self.path = path - # NOTE: pnpm requires relative workspace paths. - self.packages = set() - - def read(self): - with open(self.path) as f: - self.packages = set(yaml.load(f, Loader=yaml.CSafeLoader).get("packages", [])) - - def write(self, path=None): - if not path: - path = self.path - - with open(path, "w") as f: - data = { - "packages": list(self.packages), - } - yaml.dump(data, f, Dumper=yaml.CSafeDumper) - - def get_paths(self): - """ - Returns absolute paths of workspace packages. - :rtype: list of str - """ - dir_path = os.path.dirname(self.path) - - return [os.path.normpath(os.path.join(dir_path, pkg_path)) for pkg_path in self.packages] - - def set_from_package_json(self, package_json): - """ - Sets packages to "workspace" deps from given package.json. - :param package_json: package.json of workspace - :type package_json: PackageJson - """ - if os.path.dirname(package_json.path) != os.path.dirname(self.path): - raise TypeError( - "package.json should be in workspace directory {}, given: {}".format(os.path.dirname(self.path), package_json.path)) - - self.packages = set(path for name, path in package_json.get_workspace_dep_paths()) - # Add relative path to self. - self.packages.add(".") - - def merge(self, ws): - """ - Adds `ws`'s packages to the workspace. - :param ws: workspace to merge - :type ws: PnpmWorkspace - """ - dir_path = os.path.dirname(self.path) - ws_dir_path = os.path.dirname(ws.path) - - for p_rel_path in ws.packages: - p_path = os.path.normpath(os.path.join(ws_dir_path, p_rel_path)) - self.packages.add(os.path.relpath(p_path, dir_path)) +import os +import yaml + + +class PnpmWorkspace(object): + @classmethod + def load(cls, path): + ws = cls(path) + ws.read() + + return ws + + def __init__(self, path): + if not os.path.isabs(path): + raise TypeError("Absolute path required, given: {}".format(path)) + + self.path = path + # NOTE: pnpm requires relative workspace paths. + self.packages = set() + + def read(self): + with open(self.path) as f: + self.packages = set(yaml.load(f, Loader=yaml.CSafeLoader).get("packages", [])) + + def write(self, path=None): + if not path: + path = self.path + + with open(path, "w") as f: + data = { + "packages": list(self.packages), + } + yaml.dump(data, f, Dumper=yaml.CSafeDumper) + + def get_paths(self): + """ + Returns absolute paths of workspace packages. + :rtype: list of str + """ + dir_path = os.path.dirname(self.path) + + return [os.path.normpath(os.path.join(dir_path, pkg_path)) for pkg_path in self.packages] + + def set_from_package_json(self, package_json): + """ + Sets packages to "workspace" deps from given package.json. + :param package_json: package.json of workspace + :type package_json: PackageJson + """ + if os.path.dirname(package_json.path) != os.path.dirname(self.path): + raise TypeError( + "package.json should be in workspace directory {}, given: {}".format(os.path.dirname(self.path), package_json.path)) + + self.packages = set(path for name, path in package_json.get_workspace_dep_paths()) + # Add relative path to self. + self.packages.add(".") + + def merge(self, ws): + """ + Adds `ws`'s packages to the workspace. + :param ws: workspace to merge + :type ws: PnpmWorkspace + """ + dir_path = os.path.dirname(self.path) + ws_dir_path = os.path.dirname(ws.path) + + for p_rel_path in ws.packages: + p_path = os.path.normpath(os.path.join(ws_dir_path, p_rel_path)) + self.packages.add(os.path.relpath(p_path, dir_path)) diff --git a/build/plugins/lib/nots/package_manager/pnpm/ya.make b/build/plugins/lib/nots/package_manager/pnpm/ya.make index deeb991d66..b2f2727c3f 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/ya.make +++ b/build/plugins/lib/nots/package_manager/pnpm/ya.make @@ -1,23 +1,23 @@ -PY23_LIBRARY() - -OWNER(dankolesnikov) - -PY_SRCS( - __init__.py - lockfile.py - package_manager.py - workspace.py - utils.py -) - -PEERDIR( - build/plugins/lib/nots/package_manager/base - contrib/python/PyYAML - contrib/python/six -) - -END() - -RECURSE_FOR_TESTS( - tests -) +PY23_LIBRARY() + +OWNER(dankolesnikov) + +PY_SRCS( + __init__.py + lockfile.py + package_manager.py + workspace.py + utils.py +) + +PEERDIR( + build/plugins/lib/nots/package_manager/base + contrib/python/PyYAML + contrib/python/six +) + +END() + +RECURSE_FOR_TESTS( + tests +) diff --git a/build/plugins/lib/nots/package_manager/ya.make b/build/plugins/lib/nots/package_manager/ya.make index dd90019b72..79ee0ea175 100644 --- a/build/plugins/lib/nots/package_manager/ya.make +++ b/build/plugins/lib/nots/package_manager/ya.make @@ -1,14 +1,14 @@ -PY23_LIBRARY() - -OWNER(dankolesnikov) - -PY_SRCS( - __init__.py -) - -PEERDIR( - build/plugins/lib/nots/package_manager/base - build/plugins/lib/nots/package_manager/pnpm -) - -END() +PY23_LIBRARY() + +OWNER(dankolesnikov) + +PY_SRCS( + __init__.py +) + +PEERDIR( + build/plugins/lib/nots/package_manager/base + build/plugins/lib/nots/package_manager/pnpm +) + +END() diff --git a/build/plugins/lib/nots/typescript/__init__.py b/build/plugins/lib/nots/typescript/__init__.py index 0c724bd6e1..4684004183 100644 --- a/build/plugins/lib/nots/typescript/__init__.py +++ b/build/plugins/lib/nots/typescript/__init__.py @@ -1,7 +1,7 @@ -from .tsc_wrapper import TscWrapper, TsConfig, TsValidationError - -__all__ = [ - "TscWrapper", - "TsConfig", - "TsValidationError", -] +from .tsc_wrapper import TscWrapper, TsConfig, TsValidationError + +__all__ = [ + "TscWrapper", + "TsConfig", + "TsValidationError", +] diff --git a/build/plugins/lib/nots/typescript/tests/tsc_wrapper.py b/build/plugins/lib/nots/typescript/tests/tsc_wrapper.py index ddf309e568..b6c2845f79 100644 --- a/build/plugins/lib/nots/typescript/tests/tsc_wrapper.py +++ b/build/plugins/lib/nots/typescript/tests/tsc_wrapper.py @@ -1,168 +1,168 @@ -import pytest - -from build.plugins.lib.nots.typescript import TsConfig, TsValidationError - - -def test_ts_config_validate_valid(): - cfg = TsConfig(path="/tsconfig.json") - cfg.data = { - "compilerOptions": { - "rootDir": "./src", - "outDir": "./build", - }, - } - - cfg.validate() - - -def test_ts_config_validate_empty(): - cfg = TsConfig(path="/tsconfig.json") - - with pytest.raises(TsValidationError) as e: - cfg.validate() - - assert e.value.errors == [ - "'rootDir' option is required", - "'outDir' option is required", - ] - - -def test_ts_config_validate_invalid_common(): - cfg = TsConfig(path="/tsconfig.json") - cfg.data = { - "compilerOptions": { - "preserveSymlinks": True, - "rootDirs": [], - "outFile": "./foo.js", - }, - "references": [], - "files": [], - "include": [], - "exclude": [], - } - - with pytest.raises(TsValidationError) as e: - cfg.validate() - - assert e.value.errors == [ - "'rootDir' option is required", - "'outDir' option is required", - "'outFile' option is not supported", - "'preserveSymlinks' option is not supported due to pnpm limitations", - "'rootDirs' option is not supported, relative imports should have single root", - "'files' option is not supported, use 'include'", - "composite builds are not supported, use peerdirs in ya.make instead of 'references' option", - ] - - -def test_ts_config_validate_invalid_subdirs(): - cfg = TsConfig(path="/foo/tsconfig.json") - cfg.data = { - "compilerOptions": { - "rootDir": "/bar/src", - "outDir": "../bar/build", - }, - } - - with pytest.raises(TsValidationError) as e: - cfg.validate() - - assert e.value.errors == [ - "'rootDir' should be a subdirectory of the module", - "'outDir' should be a subdirectory of the module", - ] - - -def test_ts_config_transform(): - cfg = TsConfig(path="/tsconfig.json") - cfg.data = { - "compilerOptions": { - "rootDir": "./src", - "outDir": "./build", - "typeRoots": ["./node_modules/foo", "bar"], - }, - "include": ["src/**/*"], - } - - cfg.transform_paths( - build_path="bindir", - sources_path="srcdir", - ) - - assert cfg.data == { - "compilerOptions": { - "outDir": "bindir/build", - "rootDir": "srcdir/src", - "baseUrl": "bindir/node_modules", - "typeRoots": ["srcdir/node_modules/foo", "srcdir/bar", "bindir/node_modules/foo", "bindir/bar"] - }, - "include": ["srcdir/src/**/*"], - "exclude": [], - } - - -def test_ts_config_transform_when_root_eq_out(): - cfg = TsConfig(path="/tsconfig.json") - cfg.data = { - "compilerOptions": { - "rootDir": ".", - "outDir": ".", - }, - } - - cfg.transform_paths( - build_path="bindir", - sources_path="srcdir", - ) - - assert cfg.data == { - "compilerOptions": { - "rootDir": "srcdir", - "outDir": "bindir", - "baseUrl": "bindir/node_modules", - }, - "include": [], - "exclude": [], - } - - -def test_ts_config_transform_sets_correct_source_root(): - cfg = TsConfig(path="/tsconfig.json") - cfg.data = { - "compilerOptions": { - "rootDir": "src", - "outDir": "build", - "sourceMap": True, - }, - } - - cfg.transform_paths( - build_path="bindir", - sources_path="srcdir", - ) - - assert cfg.data == { - "compilerOptions": { - "rootDir": "srcdir/src", - "outDir": "bindir/build", - "baseUrl": "bindir/node_modules", - "sourceMap": True, - "sourceRoot": "../src", - }, - "include": [], - "exclude": [], - } - - -def test_ts_config_compiler_options(): - cfg = TsConfig(path="/tsconfig.json") - - assert cfg.compiler_option("invalid") is None - - cfg.data = { - "compilerOptions": { - "rootDir": "src", - }, - } - - assert cfg.compiler_option("rootDir") == "src" +import pytest + +from build.plugins.lib.nots.typescript import TsConfig, TsValidationError + + +def test_ts_config_validate_valid(): + cfg = TsConfig(path="/tsconfig.json") + cfg.data = { + "compilerOptions": { + "rootDir": "./src", + "outDir": "./build", + }, + } + + cfg.validate() + + +def test_ts_config_validate_empty(): + cfg = TsConfig(path="/tsconfig.json") + + with pytest.raises(TsValidationError) as e: + cfg.validate() + + assert e.value.errors == [ + "'rootDir' option is required", + "'outDir' option is required", + ] + + +def test_ts_config_validate_invalid_common(): + cfg = TsConfig(path="/tsconfig.json") + cfg.data = { + "compilerOptions": { + "preserveSymlinks": True, + "rootDirs": [], + "outFile": "./foo.js", + }, + "references": [], + "files": [], + "include": [], + "exclude": [], + } + + with pytest.raises(TsValidationError) as e: + cfg.validate() + + assert e.value.errors == [ + "'rootDir' option is required", + "'outDir' option is required", + "'outFile' option is not supported", + "'preserveSymlinks' option is not supported due to pnpm limitations", + "'rootDirs' option is not supported, relative imports should have single root", + "'files' option is not supported, use 'include'", + "composite builds are not supported, use peerdirs in ya.make instead of 'references' option", + ] + + +def test_ts_config_validate_invalid_subdirs(): + cfg = TsConfig(path="/foo/tsconfig.json") + cfg.data = { + "compilerOptions": { + "rootDir": "/bar/src", + "outDir": "../bar/build", + }, + } + + with pytest.raises(TsValidationError) as e: + cfg.validate() + + assert e.value.errors == [ + "'rootDir' should be a subdirectory of the module", + "'outDir' should be a subdirectory of the module", + ] + + +def test_ts_config_transform(): + cfg = TsConfig(path="/tsconfig.json") + cfg.data = { + "compilerOptions": { + "rootDir": "./src", + "outDir": "./build", + "typeRoots": ["./node_modules/foo", "bar"], + }, + "include": ["src/**/*"], + } + + cfg.transform_paths( + build_path="bindir", + sources_path="srcdir", + ) + + assert cfg.data == { + "compilerOptions": { + "outDir": "bindir/build", + "rootDir": "srcdir/src", + "baseUrl": "bindir/node_modules", + "typeRoots": ["srcdir/node_modules/foo", "srcdir/bar", "bindir/node_modules/foo", "bindir/bar"] + }, + "include": ["srcdir/src/**/*"], + "exclude": [], + } + + +def test_ts_config_transform_when_root_eq_out(): + cfg = TsConfig(path="/tsconfig.json") + cfg.data = { + "compilerOptions": { + "rootDir": ".", + "outDir": ".", + }, + } + + cfg.transform_paths( + build_path="bindir", + sources_path="srcdir", + ) + + assert cfg.data == { + "compilerOptions": { + "rootDir": "srcdir", + "outDir": "bindir", + "baseUrl": "bindir/node_modules", + }, + "include": [], + "exclude": [], + } + + +def test_ts_config_transform_sets_correct_source_root(): + cfg = TsConfig(path="/tsconfig.json") + cfg.data = { + "compilerOptions": { + "rootDir": "src", + "outDir": "build", + "sourceMap": True, + }, + } + + cfg.transform_paths( + build_path="bindir", + sources_path="srcdir", + ) + + assert cfg.data == { + "compilerOptions": { + "rootDir": "srcdir/src", + "outDir": "bindir/build", + "baseUrl": "bindir/node_modules", + "sourceMap": True, + "sourceRoot": "../src", + }, + "include": [], + "exclude": [], + } + + +def test_ts_config_compiler_options(): + cfg = TsConfig(path="/tsconfig.json") + + assert cfg.compiler_option("invalid") is None + + cfg.data = { + "compilerOptions": { + "rootDir": "src", + }, + } + + assert cfg.compiler_option("rootDir") == "src" diff --git a/build/plugins/lib/nots/typescript/tests/ya.make b/build/plugins/lib/nots/typescript/tests/ya.make index c51b02f8ea..f6a8e40ea1 100644 --- a/build/plugins/lib/nots/typescript/tests/ya.make +++ b/build/plugins/lib/nots/typescript/tests/ya.make @@ -1,13 +1,13 @@ -PY23_TEST() - -OWNER(dankolesnikov) - -TEST_SRCS( - tsc_wrapper.py -) - -PEERDIR( - build/plugins/lib/nots/typescript -) - -END() +PY23_TEST() + +OWNER(dankolesnikov) + +TEST_SRCS( + tsc_wrapper.py +) + +PEERDIR( + build/plugins/lib/nots/typescript +) + +END() diff --git a/build/plugins/lib/nots/typescript/tsc_wrapper.py b/build/plugins/lib/nots/typescript/tsc_wrapper.py index e7554fdb7b..9fddf6707f 100644 --- a/build/plugins/lib/nots/typescript/tsc_wrapper.py +++ b/build/plugins/lib/nots/typescript/tsc_wrapper.py @@ -1,219 +1,219 @@ -import os -import json -import shutil -import subprocess -import tarfile - -from ..package_manager import constants - - -class TsError(RuntimeError): - pass - - -class TsValidationError(TsError): - def __init__(self, path, errors): - self.path = path - self.errors = errors - - super(TsValidationError, self).__init__("Invalid tsconfig {}:\n{}".format(path, "\n".join(errors))) - - -class TsCompilationError(TsError): - def __init__(self, code, stdout, stderr): - self.code = code - self.stdout = stdout - self.stderr = stderr - - super(TsCompilationError, self).__init__("tsc exited with code {}:\n{}\n{}".format(code, stdout, stderr)) - - -class TsConfig(object): - @classmethod - def load(cls, path): - """ - :param path: tsconfig.json path - :type path: str - :rtype: TsConfig - """ - tsconfig = cls(path) - tsconfig.read() - - return tsconfig - - def __init__(self, path): - if not os.path.isabs(path): - raise TypeError("Absolute path required, given: {}".format(path)) - - self.path = path - self.data = {} - - def read(self): - try: - with open(self.path) as f: - self.data = json.load(f) - except Exception as e: - raise TsError("Failed to read tsconfig {}: {}".format(self.path, e)) - - def get_or_create_compiler_options(self): - """ - Returns ref to the "compilerOptions" dict. - :rtype: dict - """ - opts = self.data.get("compilerOptions") - if opts is None: - opts = {} - self.data["compilerOptions"] = opts - - return opts - - def compiler_option(self, name, default=None): - """ - :param name: option key - :type name: str - :param default: default value - :type default: mixed - :rtype: mixed - """ - return self.get_or_create_compiler_options().get(name, default) - - def validate(self): - """ - Checks whether the config is compatible with current toolchain. - """ - opts = self.get_or_create_compiler_options() - errors = [] - root_dir = opts.get("rootDir") - out_dir = opts.get("outDir") - config_dir = os.path.dirname(self.path) - is_mod_subdir = lambda p: not os.path.isabs(p) and os.path.normpath(os.path.join(config_dir, p)).startswith(config_dir) - - if root_dir is None: - errors.append("'rootDir' option is required") - elif not is_mod_subdir(root_dir): - errors.append("'rootDir' should be a subdirectory of the module") - - if out_dir is None: - errors.append("'outDir' option is required") - elif not is_mod_subdir(out_dir): - errors.append("'outDir' should be a subdirectory of the module") - - if opts.get("outFile") is not None: - errors.append("'outFile' option is not supported") - - if opts.get("preserveSymlinks"): - errors.append("'preserveSymlinks' option is not supported due to pnpm limitations") - - if opts.get("rootDirs") is not None: - errors.append("'rootDirs' option is not supported, relative imports should have single root") - - if self.data.get("files") is not None: - errors.append("'files' option is not supported, use 'include'") - - if self.data.get("references") is not None: - errors.append("composite builds are not supported, use peerdirs in ya.make instead of 'references' option") - - if len(errors): - raise TsValidationError(self.path, errors) - - def transform_paths(self, build_path, sources_path): - """ - Updates config with correct abs paths. - All source files/dirs will be mapped to `sources_path`, output files/dirs will be mapped to `build_path`. - :param build_path: module's build root - :type build_path: str - :param sources_path: module's source root - :type sources_path: str - """ - opts = self.get_or_create_compiler_options() - - sources_path_rel = lambda x: os.path.normpath(os.path.join(sources_path, x)) - build_path_rel = lambda x: os.path.normpath(os.path.join(build_path, x)) - - root_dir = opts["rootDir"] - out_dir = opts["outDir"] - - opts["rootDir"] = sources_path_rel(root_dir) - opts["outDir"] = build_path_rel(out_dir) - - if opts.get("typeRoots"): - opts["typeRoots"] = list(map(sources_path_rel, opts["typeRoots"])) + list(map(build_path_rel, opts["typeRoots"])) - - opts["baseUrl"] = build_path_rel("node_modules") - - self.data["include"] = list(map(sources_path_rel, self.data.get("include", []))) - self.data["exclude"] = list(map(sources_path_rel, self.data.get("exclude", []))) - - if opts.get("sourceMap"): - opts["sourceRoot"] = os.path.relpath(root_dir, out_dir) - - def write(self, path=None): - """ - :param path: tsconfig path, defaults to original path - :type path: str - """ - if path is None: - path = self.path - - with open(path, "w") as f: - json.dump(self.data, f) - - -class TscWrapper(object): - _TSCONFIG_FILENAME = "tsconfig.json" - - def __init__(self, build_root, build_path, sources_path, nodejs_bin_path, script_path, config_path): - self.build_root = build_root - self.build_path = build_path - self.sources_path = sources_path - self.nodejs_bin_path = nodejs_bin_path - self.script_path = script_path - self.config_path = config_path - - def compile(self): - self._prepare_dependencies() - config = self._build_config() - self._exec_tsc(["--build", config.path]) - - def _prepare_dependencies(self): - self._copy_package_json() - self._unpack_node_modules() - - def _copy_package_json(self): - # TODO: Validate "main" and "files" - they should include files from the output directory. - shutil.copyfile( - os.path.join(self.sources_path, constants.PACKAGE_JSON_FILENAME), - os.path.join(self.build_path, constants.PACKAGE_JSON_FILENAME), - ) - - def _unpack_node_modules(self): - nm_bundle_path = os.path.join(self.build_path, constants.NODE_MODULES_BUNDLE_FILENAME) - if os.path.isfile(nm_bundle_path): - with tarfile.open(nm_bundle_path) as tf: - tf.extractall(os.path.join(self.build_path, "node_modules")) - - def _build_config(self): - config = TsConfig.load(self.config_path) - config.validate() - config.transform_paths( - build_path=self.build_path, - sources_path=self.sources_path, - ) - - config.path = os.path.join(self.build_path, self._TSCONFIG_FILENAME) - config.write() - - return config - - def _exec_tsc(self, args): - p = subprocess.Popen( - [self.nodejs_bin_path, self.script_path] + args, - cwd=self.build_path, - stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout, stderr = p.communicate() - - if p.returncode != 0: - raise TsCompilationError(p.returncode, stdout.decode("utf-8"), stderr.decode("utf-8")) +import os +import json +import shutil +import subprocess +import tarfile + +from ..package_manager import constants + + +class TsError(RuntimeError): + pass + + +class TsValidationError(TsError): + def __init__(self, path, errors): + self.path = path + self.errors = errors + + super(TsValidationError, self).__init__("Invalid tsconfig {}:\n{}".format(path, "\n".join(errors))) + + +class TsCompilationError(TsError): + def __init__(self, code, stdout, stderr): + self.code = code + self.stdout = stdout + self.stderr = stderr + + super(TsCompilationError, self).__init__("tsc exited with code {}:\n{}\n{}".format(code, stdout, stderr)) + + +class TsConfig(object): + @classmethod + def load(cls, path): + """ + :param path: tsconfig.json path + :type path: str + :rtype: TsConfig + """ + tsconfig = cls(path) + tsconfig.read() + + return tsconfig + + def __init__(self, path): + if not os.path.isabs(path): + raise TypeError("Absolute path required, given: {}".format(path)) + + self.path = path + self.data = {} + + def read(self): + try: + with open(self.path) as f: + self.data = json.load(f) + except Exception as e: + raise TsError("Failed to read tsconfig {}: {}".format(self.path, e)) + + def get_or_create_compiler_options(self): + """ + Returns ref to the "compilerOptions" dict. + :rtype: dict + """ + opts = self.data.get("compilerOptions") + if opts is None: + opts = {} + self.data["compilerOptions"] = opts + + return opts + + def compiler_option(self, name, default=None): + """ + :param name: option key + :type name: str + :param default: default value + :type default: mixed + :rtype: mixed + """ + return self.get_or_create_compiler_options().get(name, default) + + def validate(self): + """ + Checks whether the config is compatible with current toolchain. + """ + opts = self.get_or_create_compiler_options() + errors = [] + root_dir = opts.get("rootDir") + out_dir = opts.get("outDir") + config_dir = os.path.dirname(self.path) + is_mod_subdir = lambda p: not os.path.isabs(p) and os.path.normpath(os.path.join(config_dir, p)).startswith(config_dir) + + if root_dir is None: + errors.append("'rootDir' option is required") + elif not is_mod_subdir(root_dir): + errors.append("'rootDir' should be a subdirectory of the module") + + if out_dir is None: + errors.append("'outDir' option is required") + elif not is_mod_subdir(out_dir): + errors.append("'outDir' should be a subdirectory of the module") + + if opts.get("outFile") is not None: + errors.append("'outFile' option is not supported") + + if opts.get("preserveSymlinks"): + errors.append("'preserveSymlinks' option is not supported due to pnpm limitations") + + if opts.get("rootDirs") is not None: + errors.append("'rootDirs' option is not supported, relative imports should have single root") + + if self.data.get("files") is not None: + errors.append("'files' option is not supported, use 'include'") + + if self.data.get("references") is not None: + errors.append("composite builds are not supported, use peerdirs in ya.make instead of 'references' option") + + if len(errors): + raise TsValidationError(self.path, errors) + + def transform_paths(self, build_path, sources_path): + """ + Updates config with correct abs paths. + All source files/dirs will be mapped to `sources_path`, output files/dirs will be mapped to `build_path`. + :param build_path: module's build root + :type build_path: str + :param sources_path: module's source root + :type sources_path: str + """ + opts = self.get_or_create_compiler_options() + + sources_path_rel = lambda x: os.path.normpath(os.path.join(sources_path, x)) + build_path_rel = lambda x: os.path.normpath(os.path.join(build_path, x)) + + root_dir = opts["rootDir"] + out_dir = opts["outDir"] + + opts["rootDir"] = sources_path_rel(root_dir) + opts["outDir"] = build_path_rel(out_dir) + + if opts.get("typeRoots"): + opts["typeRoots"] = list(map(sources_path_rel, opts["typeRoots"])) + list(map(build_path_rel, opts["typeRoots"])) + + opts["baseUrl"] = build_path_rel("node_modules") + + self.data["include"] = list(map(sources_path_rel, self.data.get("include", []))) + self.data["exclude"] = list(map(sources_path_rel, self.data.get("exclude", []))) + + if opts.get("sourceMap"): + opts["sourceRoot"] = os.path.relpath(root_dir, out_dir) + + def write(self, path=None): + """ + :param path: tsconfig path, defaults to original path + :type path: str + """ + if path is None: + path = self.path + + with open(path, "w") as f: + json.dump(self.data, f) + + +class TscWrapper(object): + _TSCONFIG_FILENAME = "tsconfig.json" + + def __init__(self, build_root, build_path, sources_path, nodejs_bin_path, script_path, config_path): + self.build_root = build_root + self.build_path = build_path + self.sources_path = sources_path + self.nodejs_bin_path = nodejs_bin_path + self.script_path = script_path + self.config_path = config_path + + def compile(self): + self._prepare_dependencies() + config = self._build_config() + self._exec_tsc(["--build", config.path]) + + def _prepare_dependencies(self): + self._copy_package_json() + self._unpack_node_modules() + + def _copy_package_json(self): + # TODO: Validate "main" and "files" - they should include files from the output directory. + shutil.copyfile( + os.path.join(self.sources_path, constants.PACKAGE_JSON_FILENAME), + os.path.join(self.build_path, constants.PACKAGE_JSON_FILENAME), + ) + + def _unpack_node_modules(self): + nm_bundle_path = os.path.join(self.build_path, constants.NODE_MODULES_BUNDLE_FILENAME) + if os.path.isfile(nm_bundle_path): + with tarfile.open(nm_bundle_path) as tf: + tf.extractall(os.path.join(self.build_path, "node_modules")) + + def _build_config(self): + config = TsConfig.load(self.config_path) + config.validate() + config.transform_paths( + build_path=self.build_path, + sources_path=self.sources_path, + ) + + config.path = os.path.join(self.build_path, self._TSCONFIG_FILENAME) + config.write() + + return config + + def _exec_tsc(self, args): + p = subprocess.Popen( + [self.nodejs_bin_path, self.script_path] + args, + cwd=self.build_path, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + + if p.returncode != 0: + raise TsCompilationError(p.returncode, stdout.decode("utf-8"), stderr.decode("utf-8")) diff --git a/build/plugins/lib/nots/typescript/ya.make b/build/plugins/lib/nots/typescript/ya.make index 8d739f4d9b..e83ce75e7a 100644 --- a/build/plugins/lib/nots/typescript/ya.make +++ b/build/plugins/lib/nots/typescript/ya.make @@ -1,18 +1,18 @@ -PY23_LIBRARY() - -OWNER(dankolesnikov) - -PY_SRCS( - __init__.py - tsc_wrapper.py -) - -PEERDIR( - build/plugins/lib/nots/package_manager -) - -END() - -RECURSE_FOR_TESTS( - tests -) +PY23_LIBRARY() + +OWNER(dankolesnikov) + +PY_SRCS( + __init__.py + tsc_wrapper.py +) + +PEERDIR( + build/plugins/lib/nots/package_manager +) + +END() + +RECURSE_FOR_TESTS( + tests +) diff --git a/build/plugins/lib/nots/ya.make b/build/plugins/lib/nots/ya.make index 0f965d5566..681a1dea1a 100644 --- a/build/plugins/lib/nots/ya.make +++ b/build/plugins/lib/nots/ya.make @@ -1,14 +1,14 @@ -PY23_LIBRARY() - -OWNER(dankolesnikov) - -PY_SRCS( - __init__.py -) - -PEERDIR( - build/plugins/lib/nots/package_manager - build/plugins/lib/nots/typescript -) - -END() +PY23_LIBRARY() + +OWNER(dankolesnikov) + +PY_SRCS( + __init__.py +) + +PEERDIR( + build/plugins/lib/nots/package_manager + build/plugins/lib/nots/typescript +) + +END() diff --git a/build/plugins/nots.py b/build/plugins/nots.py index b8ec11b521..5018256ddc 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -1,46 +1,46 @@ -import os - -from _common import to_yesno -from lib.nots.package_manager import manager -from lib.nots.typescript import TsConfig - - -def _create_pm(unit): - return manager( - sources_path=unit.resolve(unit.path()), - build_root="$B", - build_path=unit.path().replace("$S", "$B", 1), - contribs_path=unit.get("NPM_CONTRIBS_PATH"), - nodejs_bin_path=None, - script_path=None, - ) - - -def on_from_npm_lockfiles(unit, *args): - lf_paths = map(lambda p: unit.resolve(unit.resolve_arc_path(p)), args) - - for pkg in _create_pm(unit).extract_packages_meta_from_lockfiles(lf_paths): - unit.onfrom_npm([pkg.name, pkg.version, pkg.sky_id, pkg.integrity, pkg.integrity_algorithm, pkg.tarball_path]) - - -def onnode_modules(unit): - pm = _create_pm(unit) - unit.onpeerdir(pm.get_peer_paths_from_package_json()) - ins, outs = pm.calc_node_modules_inouts() +import os + +from _common import to_yesno +from lib.nots.package_manager import manager +from lib.nots.typescript import TsConfig + + +def _create_pm(unit): + return manager( + sources_path=unit.resolve(unit.path()), + build_root="$B", + build_path=unit.path().replace("$S", "$B", 1), + contribs_path=unit.get("NPM_CONTRIBS_PATH"), + nodejs_bin_path=None, + script_path=None, + ) + + +def on_from_npm_lockfiles(unit, *args): + lf_paths = map(lambda p: unit.resolve(unit.resolve_arc_path(p)), args) + + for pkg in _create_pm(unit).extract_packages_meta_from_lockfiles(lf_paths): + unit.onfrom_npm([pkg.name, pkg.version, pkg.sky_id, pkg.integrity, pkg.integrity_algorithm, pkg.tarball_path]) + + +def onnode_modules(unit): + pm = _create_pm(unit) + unit.onpeerdir(pm.get_peer_paths_from_package_json()) + ins, outs = pm.calc_node_modules_inouts() unit.on_node_modules(["IN"] + sorted(ins) + ["OUT"] + sorted(outs)) - - -def on_ts_configure(unit, tsconfig_path): - abs_tsconfig_path = unit.resolve(unit.resolve_arc_path(tsconfig_path)) - if not abs_tsconfig_path: - raise Exception("tsconfig not found: {}".format(tsconfig_path)) - - tsconfig = TsConfig.load(abs_tsconfig_path) - tsconfig.validate() - - unit.set(["TS_CONFIG_ROOT_DIR", tsconfig.compiler_option("rootDir")]) - unit.set(["TS_CONFIG_OUT_DIR", tsconfig.compiler_option("outDir")]) - unit.set(["TS_CONFIG_SOURCE_MAP", to_yesno(tsconfig.compiler_option("sourceMap"))]) - unit.set(["TS_CONFIG_DECLARATION", to_yesno(tsconfig.compiler_option("declaration"))]) - unit.set(["TS_CONFIG_DECLARATION_MAP", to_yesno(tsconfig.compiler_option("declarationMap"))]) - unit.set(["TS_CONFIG_PRESERVE_JSX", to_yesno(tsconfig.compiler_option("jsx") == "preserve")]) + + +def on_ts_configure(unit, tsconfig_path): + abs_tsconfig_path = unit.resolve(unit.resolve_arc_path(tsconfig_path)) + if not abs_tsconfig_path: + raise Exception("tsconfig not found: {}".format(tsconfig_path)) + + tsconfig = TsConfig.load(abs_tsconfig_path) + tsconfig.validate() + + unit.set(["TS_CONFIG_ROOT_DIR", tsconfig.compiler_option("rootDir")]) + unit.set(["TS_CONFIG_OUT_DIR", tsconfig.compiler_option("outDir")]) + unit.set(["TS_CONFIG_SOURCE_MAP", to_yesno(tsconfig.compiler_option("sourceMap"))]) + unit.set(["TS_CONFIG_DECLARATION", to_yesno(tsconfig.compiler_option("declaration"))]) + unit.set(["TS_CONFIG_DECLARATION_MAP", to_yesno(tsconfig.compiler_option("declarationMap"))]) + unit.set(["TS_CONFIG_PRESERVE_JSX", to_yesno(tsconfig.compiler_option("jsx") == "preserve")]) diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py index eb36a9d231..db4fea50bf 100755 --- a/build/scripts/fetch_from.py +++ b/build/scripts/fetch_from.py @@ -205,7 +205,7 @@ def size_printer(display_name, size): return printer -def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None): +def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None): logging.info('Downloading from url %s name %s and expected md5 %s', url, resource_file_name, expected_md5) tmp_file_name = uniq_string_generator() @@ -217,14 +217,14 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1= real_sha1 = hashlib.sha1() with open(tmp_file_name, 'wb') as fp: - copy_stream( - req.read, - fp.write, - real_md5.update, - real_sha1.update, - size_printer(resource_file_name, expected_file_size), - *([] if writers is None else writers) - ) + copy_stream( + req.read, + fp.write, + real_md5.update, + real_sha1.update, + size_printer(resource_file_name, expected_file_size), + *([] if writers is None else writers) + ) real_md5 = real_md5.hexdigest() real_file_size = os.path.getsize(tmp_file_name) diff --git a/build/scripts/fetch_from_npm.py b/build/scripts/fetch_from_npm.py index f1f9bf154f..28a1e5c929 100644 --- a/build/scripts/fetch_from_npm.py +++ b/build/scripts/fetch_from_npm.py @@ -1,104 +1,104 @@ -import os -import sys -import time -import logging -import argparse -import hashlib - -import sky -import fetch_from - - -NPM_BASEURL = "http://npm.yandex-team.ru/" - - -def parse_args(): - parser = argparse.ArgumentParser() - fetch_from.add_common_arguments(parser) - - parser.add_argument("--name", required=True) - parser.add_argument("--version", required=True) - parser.add_argument("--sky-id", required=True) - parser.add_argument("--integrity", required=True) - parser.add_argument("--integrity-algorithm", required=True) - - return parser.parse_args() - - -def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, tries=5): - """ - :param name: package name - :type name: str - :param version: package version - :type version: str - :param sky_id: sky id of tarball - :type sky_id: str - :param integrity: tarball integrity (hex) - :type integrity: str - :param integrity_algorithm: integrity algorithm (known for openssl) - :type integrity_algorithm: str - :param tries: tries count - :type tries: int - :return: path to fetched file - :rtype: str - """ - if sky.is_avaliable(): - fetcher = lambda: sky.fetch(sky_id, file_name) - else: - fetcher = lambda: _fetch_via_http(name, version, integrity, integrity_algorithm, file_name) - - fetched_file = None - exc_info = None - - for i in range(0, tries): - try: - fetched_file = fetcher() - exc_info = None - break - except Exception as e: - logging.exception(e) - exc_info = exc_info or sys.exc_info() - time.sleep(i) - - if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] - - return fetched_file - - -def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name): - # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1. - url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)]) - - hashobj = hashlib.new(integrity_algorithm) - fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update]) - - if hashobj.hexdigest() != integrity: - raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format( - integrity, - hashobj.hexdigest(), - file_name, - )) - - return fetched_file - - -def main(args): - file_name = os.path.basename(args.copy_to) - fetched_file = fetch(args.name, args.version, args.sky_id, args.integrity, args.integrity_algorithm, file_name) - fetch_from.process(fetched_file, file_name, args) - - -if __name__ == "__main__": - args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: - main(args) - except Exception as e: - logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() - - import error - sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) +import os +import sys +import time +import logging +import argparse +import hashlib + +import sky +import fetch_from + + +NPM_BASEURL = "http://npm.yandex-team.ru/" + + +def parse_args(): + parser = argparse.ArgumentParser() + fetch_from.add_common_arguments(parser) + + parser.add_argument("--name", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--sky-id", required=True) + parser.add_argument("--integrity", required=True) + parser.add_argument("--integrity-algorithm", required=True) + + return parser.parse_args() + + +def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, tries=5): + """ + :param name: package name + :type name: str + :param version: package version + :type version: str + :param sky_id: sky id of tarball + :type sky_id: str + :param integrity: tarball integrity (hex) + :type integrity: str + :param integrity_algorithm: integrity algorithm (known for openssl) + :type integrity_algorithm: str + :param tries: tries count + :type tries: int + :return: path to fetched file + :rtype: str + """ + if sky.is_avaliable(): + fetcher = lambda: sky.fetch(sky_id, file_name) + else: + fetcher = lambda: _fetch_via_http(name, version, integrity, integrity_algorithm, file_name) + + fetched_file = None + exc_info = None + + for i in range(0, tries): + try: + fetched_file = fetcher() + exc_info = None + break + except Exception as e: + logging.exception(e) + exc_info = exc_info or sys.exc_info() + time.sleep(i) + + if exc_info: + raise exc_info[0], exc_info[1], exc_info[2] + + return fetched_file + + +def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name): + # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1. + url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)]) + + hashobj = hashlib.new(integrity_algorithm) + fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update]) + + if hashobj.hexdigest() != integrity: + raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format( + integrity, + hashobj.hexdigest(), + file_name, + )) + + return fetched_file + + +def main(args): + file_name = os.path.basename(args.copy_to) + fetched_file = fetch(args.name, args.version, args.sky_id, args.integrity, args.integrity_algorithm, file_name) + fetch_from.process(fetched_file, file_name, args) + + +if __name__ == "__main__": + args = parse_args() + fetch_from.setup_logging(args, os.path.basename(__file__)) + + try: + main(args) + except Exception as e: + logging.exception(e) + print >>sys.stderr, open(args.abs_log_path).read() + sys.stderr.flush() + + import error + sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/sky.py b/build/scripts/sky.py index 15e95e76c0..b703af7ed1 100644 --- a/build/scripts/sky.py +++ b/build/scripts/sky.py @@ -1,43 +1,43 @@ -import logging -import os -import subprocess - -import fetch_from - - -class UnsupportedProtocolException(Exception): - pass - - -def executable_path(): - return "/usr/local/bin/sky" - - -def is_avaliable(): - if not os.path.exists(executable_path()): - return False - try: - subprocess.check_output([executable_path(), "--version"]) - return True - except subprocess.CalledProcessError: - return False - except OSError: - return False - - -def fetch(skynet_id, file_name, timeout=None): - if not is_avaliable(): - raise UnsupportedProtocolException("Skynet is not available") - - target_dir = os.path.abspath(fetch_from.uniq_string_generator()) - os.mkdir(target_dir) - - cmd_args = [executable_path(), "get", "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id] - if timeout is not None: - cmd_args += ["--timeout", str(timeout)] - - logging.info("Call skynet with args: %s", cmd_args) - stdout = subprocess.check_output(cmd_args).strip() - logging.debug("Skynet call with args %s is finished, result is %s", cmd_args, stdout) - - return os.path.join(target_dir, file_name) +import logging +import os +import subprocess + +import fetch_from + + +class UnsupportedProtocolException(Exception): + pass + + +def executable_path(): + return "/usr/local/bin/sky" + + +def is_avaliable(): + if not os.path.exists(executable_path()): + return False + try: + subprocess.check_output([executable_path(), "--version"]) + return True + except subprocess.CalledProcessError: + return False + except OSError: + return False + + +def fetch(skynet_id, file_name, timeout=None): + if not is_avaliable(): + raise UnsupportedProtocolException("Skynet is not available") + + target_dir = os.path.abspath(fetch_from.uniq_string_generator()) + os.mkdir(target_dir) + + cmd_args = [executable_path(), "get", "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id] + if timeout is not None: + cmd_args += ["--timeout", str(timeout)] + + logging.info("Call skynet with args: %s", cmd_args) + stdout = subprocess.check_output(cmd_args).strip() + logging.debug("Skynet call with args %s is finished, result is %s", cmd_args, stdout) + + return os.path.join(target_dir, file_name) diff --git a/build/scripts/ya.make b/build/scripts/ya.make index 068e4c15f1..710165e40d 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -32,7 +32,7 @@ TEST_SRCS( fetch_from.py fetch_from_external.py fetch_from_mds.py - fetch_from_npm.py + fetch_from_npm.py fetch_from_sandbox.py fetch_resource.py filter_zip.py @@ -75,7 +75,7 @@ TEST_SRCS( run_llvm_dsymutil.py run_msvc_wine.py run_tool.py - sky.py + sky.py stdout2stderr.py symlink.py tar_directory.py diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 04e523e518..081833998b 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -35,7 +35,7 @@ SO_OTPUTS=no @import "${CONF_ROOT}/conf/rules.conf" @import "${CONF_ROOT}/conf/sysincl.conf" @import "${CONF_ROOT}/conf/license.conf" -@import "${CONF_ROOT}/conf/ts.conf" +@import "${CONF_ROOT}/conf/ts.conf" @import "${CONF_ROOT}/conf/project_specific/other.conf" diff --git a/tools/ya.make b/tools/ya.make index 7c0fcb69a9..51a6b8b426 100644 --- a/tools/ya.make +++ b/tools/ya.make @@ -83,7 +83,7 @@ RECURSE( nodeiter_test nodeiter_test/tests normalize_requests - nots + nots oauth_token pgmigrate pire |