aboutsummaryrefslogtreecommitdiffstats
path: root/build/scripts
diff options
context:
space:
mode:
authordankolesnikov <dankolesnikov@yandex-team.ru>2022-02-10 16:51:07 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:51:07 +0300
commit98174ab8f0e6dbed8894c8226f62cac4bf36171d (patch)
tree00cab7b3b62da0fe104a2a99c2886064cc0b0d63 /build/scripts
parent2e8363373770594fa3e83f1410d513cff82abb30 (diff)
downloadydb-98174ab8f0e6dbed8894c8226f62cac4bf36171d.tar.gz
Restoring authorship annotation for <dankolesnikov@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'build/scripts')
-rwxr-xr-xbuild/scripts/fetch_from.py18
-rw-r--r--build/scripts/fetch_from_npm.py208
-rw-r--r--build/scripts/sky.py86
-rw-r--r--build/scripts/ya.make4
4 files changed, 158 insertions, 158 deletions
diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py
index db4fea50bf..eb36a9d231 100755
--- a/build/scripts/fetch_from.py
+++ b/build/scripts/fetch_from.py
@@ -205,7 +205,7 @@ def size_printer(display_name, size):
return printer
-def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None):
+def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None):
logging.info('Downloading from url %s name %s and expected md5 %s', url, resource_file_name, expected_md5)
tmp_file_name = uniq_string_generator()
@@ -217,14 +217,14 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=
real_sha1 = hashlib.sha1()
with open(tmp_file_name, 'wb') as fp:
- copy_stream(
- req.read,
- fp.write,
- real_md5.update,
- real_sha1.update,
- size_printer(resource_file_name, expected_file_size),
- *([] if writers is None else writers)
- )
+ copy_stream(
+ req.read,
+ fp.write,
+ real_md5.update,
+ real_sha1.update,
+ size_printer(resource_file_name, expected_file_size),
+ *([] if writers is None else writers)
+ )
real_md5 = real_md5.hexdigest()
real_file_size = os.path.getsize(tmp_file_name)
diff --git a/build/scripts/fetch_from_npm.py b/build/scripts/fetch_from_npm.py
index 28a1e5c929..f1f9bf154f 100644
--- a/build/scripts/fetch_from_npm.py
+++ b/build/scripts/fetch_from_npm.py
@@ -1,104 +1,104 @@
-import os
-import sys
-import time
-import logging
-import argparse
-import hashlib
-
-import sky
-import fetch_from
-
-
-NPM_BASEURL = "http://npm.yandex-team.ru/"
-
-
-def parse_args():
- parser = argparse.ArgumentParser()
- fetch_from.add_common_arguments(parser)
-
- parser.add_argument("--name", required=True)
- parser.add_argument("--version", required=True)
- parser.add_argument("--sky-id", required=True)
- parser.add_argument("--integrity", required=True)
- parser.add_argument("--integrity-algorithm", required=True)
-
- return parser.parse_args()
-
-
-def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, tries=5):
- """
- :param name: package name
- :type name: str
- :param version: package version
- :type version: str
- :param sky_id: sky id of tarball
- :type sky_id: str
- :param integrity: tarball integrity (hex)
- :type integrity: str
- :param integrity_algorithm: integrity algorithm (known for openssl)
- :type integrity_algorithm: str
- :param tries: tries count
- :type tries: int
- :return: path to fetched file
- :rtype: str
- """
- if sky.is_avaliable():
- fetcher = lambda: sky.fetch(sky_id, file_name)
- else:
- fetcher = lambda: _fetch_via_http(name, version, integrity, integrity_algorithm, file_name)
-
- fetched_file = None
- exc_info = None
-
- for i in range(0, tries):
- try:
- fetched_file = fetcher()
- exc_info = None
- break
- except Exception as e:
- logging.exception(e)
- exc_info = exc_info or sys.exc_info()
- time.sleep(i)
-
- if exc_info:
- raise exc_info[0], exc_info[1], exc_info[2]
-
- return fetched_file
-
-
-def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name):
- # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1.
- url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)])
-
- hashobj = hashlib.new(integrity_algorithm)
- fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update])
-
- if hashobj.hexdigest() != integrity:
- raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format(
- integrity,
- hashobj.hexdigest(),
- file_name,
- ))
-
- return fetched_file
-
-
-def main(args):
- file_name = os.path.basename(args.copy_to)
- fetched_file = fetch(args.name, args.version, args.sky_id, args.integrity, args.integrity_algorithm, file_name)
- fetch_from.process(fetched_file, file_name, args)
-
-
-if __name__ == "__main__":
- args = parse_args()
- fetch_from.setup_logging(args, os.path.basename(__file__))
-
- try:
- main(args)
- except Exception as e:
- logging.exception(e)
- print >>sys.stderr, open(args.abs_log_path).read()
- sys.stderr.flush()
-
- import error
- sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1)
+import os
+import sys
+import time
+import logging
+import argparse
+import hashlib
+
+import sky
+import fetch_from
+
+
+NPM_BASEURL = "http://npm.yandex-team.ru/"
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ fetch_from.add_common_arguments(parser)
+
+ parser.add_argument("--name", required=True)
+ parser.add_argument("--version", required=True)
+ parser.add_argument("--sky-id", required=True)
+ parser.add_argument("--integrity", required=True)
+ parser.add_argument("--integrity-algorithm", required=True)
+
+ return parser.parse_args()
+
+
+def fetch(name, version, sky_id, integrity, integrity_algorithm, file_name, tries=5):
+ """
+ :param name: package name
+ :type name: str
+ :param version: package version
+ :type version: str
+ :param sky_id: sky id of tarball
+ :type sky_id: str
+ :param integrity: tarball integrity (hex)
+ :type integrity: str
+ :param integrity_algorithm: integrity algorithm (known for openssl)
+ :type integrity_algorithm: str
+ :param tries: tries count
+ :type tries: int
+ :return: path to fetched file
+ :rtype: str
+ """
+ if sky.is_avaliable():
+ fetcher = lambda: sky.fetch(sky_id, file_name)
+ else:
+ fetcher = lambda: _fetch_via_http(name, version, integrity, integrity_algorithm, file_name)
+
+ fetched_file = None
+ exc_info = None
+
+ for i in range(0, tries):
+ try:
+ fetched_file = fetcher()
+ exc_info = None
+ break
+ except Exception as e:
+ logging.exception(e)
+ exc_info = exc_info or sys.exc_info()
+ time.sleep(i)
+
+ if exc_info:
+ raise exc_info[0], exc_info[1], exc_info[2]
+
+ return fetched_file
+
+
+def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name):
+ # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1.
+ url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)])
+
+ hashobj = hashlib.new(integrity_algorithm)
+ fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update])
+
+ if hashobj.hexdigest() != integrity:
+ raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format(
+ integrity,
+ hashobj.hexdigest(),
+ file_name,
+ ))
+
+ return fetched_file
+
+
+def main(args):
+ file_name = os.path.basename(args.copy_to)
+ fetched_file = fetch(args.name, args.version, args.sky_id, args.integrity, args.integrity_algorithm, file_name)
+ fetch_from.process(fetched_file, file_name, args)
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ fetch_from.setup_logging(args, os.path.basename(__file__))
+
+ try:
+ main(args)
+ except Exception as e:
+ logging.exception(e)
+ print >>sys.stderr, open(args.abs_log_path).read()
+ sys.stderr.flush()
+
+ import error
+ sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1)
diff --git a/build/scripts/sky.py b/build/scripts/sky.py
index b703af7ed1..15e95e76c0 100644
--- a/build/scripts/sky.py
+++ b/build/scripts/sky.py
@@ -1,43 +1,43 @@
-import logging
-import os
-import subprocess
-
-import fetch_from
-
-
-class UnsupportedProtocolException(Exception):
- pass
-
-
-def executable_path():
- return "/usr/local/bin/sky"
-
-
-def is_avaliable():
- if not os.path.exists(executable_path()):
- return False
- try:
- subprocess.check_output([executable_path(), "--version"])
- return True
- except subprocess.CalledProcessError:
- return False
- except OSError:
- return False
-
-
-def fetch(skynet_id, file_name, timeout=None):
- if not is_avaliable():
- raise UnsupportedProtocolException("Skynet is not available")
-
- target_dir = os.path.abspath(fetch_from.uniq_string_generator())
- os.mkdir(target_dir)
-
- cmd_args = [executable_path(), "get", "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id]
- if timeout is not None:
- cmd_args += ["--timeout", str(timeout)]
-
- logging.info("Call skynet with args: %s", cmd_args)
- stdout = subprocess.check_output(cmd_args).strip()
- logging.debug("Skynet call with args %s is finished, result is %s", cmd_args, stdout)
-
- return os.path.join(target_dir, file_name)
+import logging
+import os
+import subprocess
+
+import fetch_from
+
+
+class UnsupportedProtocolException(Exception):
+ pass
+
+
+def executable_path():
+ return "/usr/local/bin/sky"
+
+
+def is_avaliable():
+ if not os.path.exists(executable_path()):
+ return False
+ try:
+ subprocess.check_output([executable_path(), "--version"])
+ return True
+ except subprocess.CalledProcessError:
+ return False
+ except OSError:
+ return False
+
+
+def fetch(skynet_id, file_name, timeout=None):
+ if not is_avaliable():
+ raise UnsupportedProtocolException("Skynet is not available")
+
+ target_dir = os.path.abspath(fetch_from.uniq_string_generator())
+ os.mkdir(target_dir)
+
+ cmd_args = [executable_path(), "get", "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id]
+ if timeout is not None:
+ cmd_args += ["--timeout", str(timeout)]
+
+ logging.info("Call skynet with args: %s", cmd_args)
+ stdout = subprocess.check_output(cmd_args).strip()
+ logging.debug("Skynet call with args %s is finished, result is %s", cmd_args, stdout)
+
+ return os.path.join(target_dir, file_name)
diff --git a/build/scripts/ya.make b/build/scripts/ya.make
index 710165e40d..068e4c15f1 100644
--- a/build/scripts/ya.make
+++ b/build/scripts/ya.make
@@ -32,7 +32,7 @@ TEST_SRCS(
fetch_from.py
fetch_from_external.py
fetch_from_mds.py
- fetch_from_npm.py
+ fetch_from_npm.py
fetch_from_sandbox.py
fetch_resource.py
filter_zip.py
@@ -75,7 +75,7 @@ TEST_SRCS(
run_llvm_dsymutil.py
run_msvc_wine.py
run_tool.py
- sky.py
+ sky.py
stdout2stderr.py
symlink.py
tar_directory.py