diff options
author | Aleksandr <ivansduck@gmail.com> | 2022-02-10 16:47:52 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:52 +0300 |
commit | b05913d1c3c02a773578bceb7285084d2933ae86 (patch) | |
tree | c0748b5dcbade83af788c0abfa89c0383d6b779c /build/scripts/fetch_from.py | |
parent | ea6c5b7f172becca389cacaff7d5f45f6adccbe6 (diff) | |
download | ydb-b05913d1c3c02a773578bceb7285084d2933ae86.tar.gz |
Restoring authorship annotation for Aleksandr <ivansduck@gmail.com>. Commit 2 of 2.
Diffstat (limited to 'build/scripts/fetch_from.py')
-rwxr-xr-x | build/scripts/fetch_from.py | 114 |
1 files changed, 57 insertions, 57 deletions
diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py index bbca65219f..db4fea50bf 100755 --- a/build/scripts/fetch_from.py +++ b/build/scripts/fetch_from.py @@ -1,19 +1,19 @@ -import datetime as dt -import errno +import datetime as dt +import errno import hashlib -import json -import logging -import os +import json +import logging +import os import platform import random -import shutil -import socket +import shutil +import socket import string import sys -import tarfile -import urllib2 +import tarfile +import urllib2 -import retry +import retry def make_user_agent(): @@ -29,7 +29,7 @@ def add_common_arguments(parser): parser.add_argument('--executable', action='store_true', help='make outputs executable') parser.add_argument('--log-path') parser.add_argument('-v', '--verbose', action='store_true', default=os.environ.get('YA_VERBOSE_FETCHER'), help='increase stderr verbosity') - parser.add_argument('outputs', nargs='*', default=[]) + parser.add_argument('outputs', nargs='*', default=[]) def ensure_dir(path): @@ -37,7 +37,7 @@ def ensure_dir(path): os.makedirs(path) -# Reference code: library/python/fs/__init__.py +# Reference code: library/python/fs/__init__.py def hardlink_or_copy(src, dst): ensure_dir(os.path.dirname(dst)) @@ -49,23 +49,23 @@ def hardlink_or_copy(src, dst): except OSError as e: if e.errno == errno.EEXIST: return - elif e.errno in (errno.EXDEV, errno.EMLINK, errno.EINVAL, errno.EACCES): - sys.stderr.write("Can't make hardlink (errno={}) - fallback to copy: {} -> {}\n".format(e.errno, src, dst)) + elif e.errno in (errno.EXDEV, errno.EMLINK, errno.EINVAL, errno.EACCES): + sys.stderr.write("Can't make hardlink (errno={}) - fallback to copy: {} -> {}\n".format(e.errno, src, dst)) shutil.copy(src, dst) else: raise -def rename_or_copy_and_remove(src, dst): +def rename_or_copy_and_remove(src, dst): ensure_dir(os.path.dirname(dst)) - - try: - os.rename(src, dst) - except OSError: + + try: + os.rename(src, dst) + except OSError: shutil.copy(src, dst) - os.remove(src) - - + os.remove(src) + + class BadChecksumFetchError(Exception): pass @@ -114,17 +114,17 @@ def is_temporary(e): def is_broken(e): return isinstance(e, urllib2.HTTPError) and e.code in (410, 404) - if is_broken(e): - return False + if is_broken(e): + return False + + if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, urllib2.URLError, socket.error)): + return True + + import error + + return error.is_temporary_error(e) - if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, urllib2.URLError, socket.error)): - return True - import error - - return error.is_temporary_error(e) - - def uniq_string_generator(size=6, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) @@ -292,28 +292,28 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1= return tmp_file_name -def chmod(filename, mode): +def chmod(filename, mode): if platform.system().lower() == 'windows': # https://docs.microsoft.com/en-us/windows/win32/fileio/hard-links-and-junctions: # hard to reset read-only attribute for removal if there are multiple hardlinks return - stat = os.stat(filename) - if stat.st_mode & 0o777 != mode: - try: - os.chmod(filename, mode) - except OSError: + stat = os.stat(filename) + if stat.st_mode & 0o777 != mode: + try: + os.chmod(filename, mode) + except OSError: import pwd - sys.stderr.write("{} st_mode: {} pwuid: {}\n".format(filename, stat.st_mode, pwd.getpwuid(os.stat(filename).st_uid))) - raise - - + sys.stderr.write("{} st_mode: {} pwuid: {}\n".format(filename, stat.st_mode, pwd.getpwuid(os.stat(filename).st_uid))) + raise + + def process(fetched_file, file_name, args, remove=True): assert len(args.rename) <= len(args.outputs), ( 'too few outputs to rename', args.rename, 'into', args.outputs) - # Forbid changes to the loaded resource - chmod(fetched_file, 0o444) - + # Forbid changes to the loaded resource + chmod(fetched_file, 0o444) + if not os.path.isfile(fetched_file): raise ResourceIsDirectoryError('Resource must be a file, not a directory: %s' % fetched_file) @@ -332,16 +332,16 @@ def process(fetched_file, file_name, args, remove=True): if args.untar_to: ensure_dir(args.untar_to) - # Extract only requested files + # Extract only requested files try: with tarfile.open(fetched_file, mode='r:*') as tar: - inputs = set(map(os.path.normpath, args.rename + args.outputs[len(args.rename):])) - members = [entry for entry in tar if os.path.normpath(os.path.join(args.untar_to, entry.name)) in inputs] - tar.extractall(args.untar_to, members=members) - # Forbid changes to the loaded resource data - for root, _, files in os.walk(args.untar_to): - for filename in files: - chmod(os.path.join(root, filename), 0o444) + inputs = set(map(os.path.normpath, args.rename + args.outputs[len(args.rename):])) + members = [entry for entry in tar if os.path.normpath(os.path.join(args.untar_to, entry.name)) in inputs] + tar.extractall(args.untar_to, members=members) + # Forbid changes to the loaded resource data + for root, _, files in os.walk(args.untar_to): + for filename in files: + chmod(os.path.join(root, filename), 0o444) except tarfile.ReadError as e: logging.exception(e) raise ResourceUnpackingError('File {} cannot be untared'.format(fetched_file)) @@ -354,12 +354,12 @@ def process(fetched_file, file_name, args, remove=True): hardlink_or_copy(src, dst) else: logging.info('Renaming %s to %s', src, dst) - if os.path.exists(dst): - raise ResourceUnpackingError("Target file already exists ({} -> {})".format(src, dst)) + if os.path.exists(dst): + raise ResourceUnpackingError("Target file already exists ({} -> {})".format(src, dst)) if remove: rename_or_copy_and_remove(src, dst) else: - hardlink_or_copy(src, dst) + hardlink_or_copy(src, dst) for path in args.outputs: if not os.path.exists(path): @@ -367,9 +367,9 @@ def process(fetched_file, file_name, args, remove=True): if not os.path.isfile(path): raise OutputIsDirectoryError('Output must be a file, not a directory: %s' % os.path.abspath(path)) if args.executable: - chmod(path, os.stat(path).st_mode | 0o111) + chmod(path, os.stat(path).st_mode | 0o111) if os.path.abspath(path) == os.path.abspath(fetched_file): remove = False - + if remove: os.remove(fetched_file) |