diff options
author | dankolesnikov <dankolesnikov@yandex-team.ru> | 2022-02-10 16:51:07 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:51:07 +0300 |
commit | 98174ab8f0e6dbed8894c8226f62cac4bf36171d (patch) | |
tree | 00cab7b3b62da0fe104a2a99c2886064cc0b0d63 /build/scripts | |
parent | 2e8363373770594fa3e83f1410d513cff82abb30 (diff) | |
download | ydb-98174ab8f0e6dbed8894c8226f62cac4bf36171d.tar.gz |
Restoring authorship annotation for <dankolesnikov@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'build/scripts')
-rwxr-xr-x | build/scripts/fetch_from.py | 18 | ||||
-rw-r--r-- | build/scripts/fetch_from_npm.py | 208 | ||||
-rw-r--r-- | build/scripts/sky.py | 86 | ||||
-rw-r--r-- | build/scripts/ya.make | 4 |
4 files changed, 158 insertions, 158 deletions
diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py index db4fea50bf..eb36a9d231 100755 --- a/build/scripts/fetch_from.py +++ b/build/scripts/fetch_from.py @@ -205,7 +205,7 @@ def size_printer(display_name, size): return printer -def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None): +def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None): logging.info('Downloading from url %s name %s and expected md5 %s', url, resource_file_name, expected_md5) tmp_file_name = uniq_string_generator() @@ -217,14 +217,14 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1= real_sha1 = hashlib.sha1() with open(tmp_file_name, 'wb') as fp: - copy_stream( - req.read, - fp.write, - real_md5.update, - real_sha1.update, - size_printer(resource_file_name, expected_file_size), - *([] if writers is None else writers) - ) + copy_stream( + req.read, + fp.write, + real_md5.update, + real_sha1.update, + size_printer(resource_file_name, expected_file_size), + *([] if writers is None else writers) + ) real_md5 = real_md5.hexdigest() real_file_size = os.path.getsize(tmp_file_name) diff --git a/build/scripts/fetch_from_npm.py b/build/scripts/fetch_from_npm.py index 28a1e5c929..f1f9bf154f 100644 --- a/build/scripts/fetch_from_npm.py +++ b/build/scripts/fetch_from_npm.py @@ -1,104 +1,104 @@ -import os -import sys -import time -import logging -import argparse -import hashlib - -import sky -import fetch_from - - -NPM_BASEURL = "http://npm.yandex-team.ru/" - - -def parse_args(): - parser = argparse.ArgumentParser() - fetch_from.add_common_arguments(parser) - - parser.add_argument("--name", required=True) - parser.add_argument("--version", required=True) - parser.add_argument("--sky-id", required=True) - parser.add_argument("--integrity", required=True) - parser.add_argument("--integrity-algorithm", required=True) - - return parser.parse_args() - - -def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, tries=5): - """ - :param name: package name - :type name: str - :param version: package version - :type version: str - :param sky_id: sky id of tarball - :type sky_id: str - :param integrity: tarball integrity (hex) - :type integrity: str - :param integrity_algorithm: integrity algorithm (known for openssl) - :type integrity_algorithm: str - :param tries: tries count - :type tries: int - :return: path to fetched file - :rtype: str - """ - if sky.is_avaliable(): - fetcher = lambda: sky.fetch(sky_id, file_name) - else: - fetcher = lambda: _fetch_via_http(name, version, integrity, integrity_algorithm, file_name) - - fetched_file = None - exc_info = None - - for i in range(0, tries): - try: - fetched_file = fetcher() - exc_info = None - break - except Exception as e: - logging.exception(e) - exc_info = exc_info or sys.exc_info() - time.sleep(i) - - if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] - - return fetched_file - - -def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name): - # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1. - url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)]) - - hashobj = hashlib.new(integrity_algorithm) - fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update]) - - if hashobj.hexdigest() != integrity: - raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format( - integrity, - hashobj.hexdigest(), - file_name, - )) - - return fetched_file - - -def main(args): - file_name = os.path.basename(args.copy_to) - fetched_file = fetch(args.name, args.version, args.sky_id, args.integrity, args.integrity_algorithm, file_name) - fetch_from.process(fetched_file, file_name, args) - - -if __name__ == "__main__": - args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: - main(args) - except Exception as e: - logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() - - import error - sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) +import os +import sys +import time +import logging +import argparse +import hashlib + +import sky +import fetch_from + + +NPM_BASEURL = "http://npm.yandex-team.ru/" + + +def parse_args(): + parser = argparse.ArgumentParser() + fetch_from.add_common_arguments(parser) + + parser.add_argument("--name", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--sky-id", required=True) + parser.add_argument("--integrity", required=True) + parser.add_argument("--integrity-algorithm", required=True) + + return parser.parse_args() + + +def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, tries=5): + """ + :param name: package name + :type name: str + :param version: package version + :type version: str + :param sky_id: sky id of tarball + :type sky_id: str + :param integrity: tarball integrity (hex) + :type integrity: str + :param integrity_algorithm: integrity algorithm (known for openssl) + :type integrity_algorithm: str + :param tries: tries count + :type tries: int + :return: path to fetched file + :rtype: str + """ + if sky.is_avaliable(): + fetcher = lambda: sky.fetch(sky_id, file_name) + else: + fetcher = lambda: _fetch_via_http(name, version, integrity, integrity_algorithm, file_name) + + fetched_file = None + exc_info = None + + for i in range(0, tries): + try: + fetched_file = fetcher() + exc_info = None + break + except Exception as e: + logging.exception(e) + exc_info = exc_info or sys.exc_info() + time.sleep(i) + + if exc_info: + raise exc_info[0], exc_info[1], exc_info[2] + + return fetched_file + + +def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name): + # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1. + url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)]) + + hashobj = hashlib.new(integrity_algorithm) + fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update]) + + if hashobj.hexdigest() != integrity: + raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format( + integrity, + hashobj.hexdigest(), + file_name, + )) + + return fetched_file + + +def main(args): + file_name = os.path.basename(args.copy_to) + fetched_file = fetch(args.name, args.version, args.sky_id, args.integrity, args.integrity_algorithm, file_name) + fetch_from.process(fetched_file, file_name, args) + + +if __name__ == "__main__": + args = parse_args() + fetch_from.setup_logging(args, os.path.basename(__file__)) + + try: + main(args) + except Exception as e: + logging.exception(e) + print >>sys.stderr, open(args.abs_log_path).read() + sys.stderr.flush() + + import error + sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/sky.py b/build/scripts/sky.py index b703af7ed1..15e95e76c0 100644 --- a/build/scripts/sky.py +++ b/build/scripts/sky.py @@ -1,43 +1,43 @@ -import logging -import os -import subprocess - -import fetch_from - - -class UnsupportedProtocolException(Exception): - pass - - -def executable_path(): - return "/usr/local/bin/sky" - - -def is_avaliable(): - if not os.path.exists(executable_path()): - return False - try: - subprocess.check_output([executable_path(), "--version"]) - return True - except subprocess.CalledProcessError: - return False - except OSError: - return False - - -def fetch(skynet_id, file_name, timeout=None): - if not is_avaliable(): - raise UnsupportedProtocolException("Skynet is not available") - - target_dir = os.path.abspath(fetch_from.uniq_string_generator()) - os.mkdir(target_dir) - - cmd_args = [executable_path(), "get", "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id] - if timeout is not None: - cmd_args += ["--timeout", str(timeout)] - - logging.info("Call skynet with args: %s", cmd_args) - stdout = subprocess.check_output(cmd_args).strip() - logging.debug("Skynet call with args %s is finished, result is %s", cmd_args, stdout) - - return os.path.join(target_dir, file_name) +import logging +import os +import subprocess + +import fetch_from + + +class UnsupportedProtocolException(Exception): + pass + + +def executable_path(): + return "/usr/local/bin/sky" + + +def is_avaliable(): + if not os.path.exists(executable_path()): + return False + try: + subprocess.check_output([executable_path(), "--version"]) + return True + except subprocess.CalledProcessError: + return False + except OSError: + return False + + +def fetch(skynet_id, file_name, timeout=None): + if not is_avaliable(): + raise UnsupportedProtocolException("Skynet is not available") + + target_dir = os.path.abspath(fetch_from.uniq_string_generator()) + os.mkdir(target_dir) + + cmd_args = [executable_path(), "get", "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id] + if timeout is not None: + cmd_args += ["--timeout", str(timeout)] + + logging.info("Call skynet with args: %s", cmd_args) + stdout = subprocess.check_output(cmd_args).strip() + logging.debug("Skynet call with args %s is finished, result is %s", cmd_args, stdout) + + return os.path.join(target_dir, file_name) diff --git a/build/scripts/ya.make b/build/scripts/ya.make index 710165e40d..068e4c15f1 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -32,7 +32,7 @@ TEST_SRCS( fetch_from.py fetch_from_external.py fetch_from_mds.py - fetch_from_npm.py + fetch_from_npm.py fetch_from_sandbox.py fetch_resource.py filter_zip.py @@ -75,7 +75,7 @@ TEST_SRCS( run_llvm_dsymutil.py run_msvc_wine.py run_tool.py - sky.py + sky.py stdout2stderr.py symlink.py tar_directory.py |