aboutsummaryrefslogtreecommitdiffstats
path: root/build/scripts/fetch_from.py
diff options
context:
space:
mode:
authorAleksandr <ivansduck@gmail.com>2022-02-10 16:47:52 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:47:52 +0300
commitb05913d1c3c02a773578bceb7285084d2933ae86 (patch)
treec0748b5dcbade83af788c0abfa89c0383d6b779c /build/scripts/fetch_from.py
parentea6c5b7f172becca389cacaff7d5f45f6adccbe6 (diff)
downloadydb-b05913d1c3c02a773578bceb7285084d2933ae86.tar.gz
Restoring authorship annotation for Aleksandr <ivansduck@gmail.com>. Commit 2 of 2.
Diffstat (limited to 'build/scripts/fetch_from.py')
-rwxr-xr-xbuild/scripts/fetch_from.py114
1 files changed, 57 insertions, 57 deletions
diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py
index bbca65219f..db4fea50bf 100755
--- a/build/scripts/fetch_from.py
+++ b/build/scripts/fetch_from.py
@@ -1,19 +1,19 @@
-import datetime as dt
-import errno
+import datetime as dt
+import errno
import hashlib
-import json
-import logging
-import os
+import json
+import logging
+import os
import platform
import random
-import shutil
-import socket
+import shutil
+import socket
import string
import sys
-import tarfile
-import urllib2
+import tarfile
+import urllib2
-import retry
+import retry
def make_user_agent():
@@ -29,7 +29,7 @@ def add_common_arguments(parser):
parser.add_argument('--executable', action='store_true', help='make outputs executable')
parser.add_argument('--log-path')
parser.add_argument('-v', '--verbose', action='store_true', default=os.environ.get('YA_VERBOSE_FETCHER'), help='increase stderr verbosity')
- parser.add_argument('outputs', nargs='*', default=[])
+ parser.add_argument('outputs', nargs='*', default=[])
def ensure_dir(path):
@@ -37,7 +37,7 @@ def ensure_dir(path):
os.makedirs(path)
-# Reference code: library/python/fs/__init__.py
+# Reference code: library/python/fs/__init__.py
def hardlink_or_copy(src, dst):
ensure_dir(os.path.dirname(dst))
@@ -49,23 +49,23 @@ def hardlink_or_copy(src, dst):
except OSError as e:
if e.errno == errno.EEXIST:
return
- elif e.errno in (errno.EXDEV, errno.EMLINK, errno.EINVAL, errno.EACCES):
- sys.stderr.write("Can't make hardlink (errno={}) - fallback to copy: {} -> {}\n".format(e.errno, src, dst))
+ elif e.errno in (errno.EXDEV, errno.EMLINK, errno.EINVAL, errno.EACCES):
+ sys.stderr.write("Can't make hardlink (errno={}) - fallback to copy: {} -> {}\n".format(e.errno, src, dst))
shutil.copy(src, dst)
else:
raise
-def rename_or_copy_and_remove(src, dst):
+def rename_or_copy_and_remove(src, dst):
ensure_dir(os.path.dirname(dst))
-
- try:
- os.rename(src, dst)
- except OSError:
+
+ try:
+ os.rename(src, dst)
+ except OSError:
shutil.copy(src, dst)
- os.remove(src)
-
-
+ os.remove(src)
+
+
class BadChecksumFetchError(Exception):
pass
@@ -114,17 +114,17 @@ def is_temporary(e):
def is_broken(e):
return isinstance(e, urllib2.HTTPError) and e.code in (410, 404)
- if is_broken(e):
- return False
+ if is_broken(e):
+ return False
+
+ if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, urllib2.URLError, socket.error)):
+ return True
+
+ import error
+
+ return error.is_temporary_error(e)
- if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, urllib2.URLError, socket.error)):
- return True
- import error
-
- return error.is_temporary_error(e)
-
-
def uniq_string_generator(size=6, chars=string.ascii_lowercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
@@ -292,28 +292,28 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=
return tmp_file_name
-def chmod(filename, mode):
+def chmod(filename, mode):
if platform.system().lower() == 'windows':
# https://docs.microsoft.com/en-us/windows/win32/fileio/hard-links-and-junctions:
# hard to reset read-only attribute for removal if there are multiple hardlinks
return
- stat = os.stat(filename)
- if stat.st_mode & 0o777 != mode:
- try:
- os.chmod(filename, mode)
- except OSError:
+ stat = os.stat(filename)
+ if stat.st_mode & 0o777 != mode:
+ try:
+ os.chmod(filename, mode)
+ except OSError:
import pwd
- sys.stderr.write("{} st_mode: {} pwuid: {}\n".format(filename, stat.st_mode, pwd.getpwuid(os.stat(filename).st_uid)))
- raise
-
-
+ sys.stderr.write("{} st_mode: {} pwuid: {}\n".format(filename, stat.st_mode, pwd.getpwuid(os.stat(filename).st_uid)))
+ raise
+
+
def process(fetched_file, file_name, args, remove=True):
assert len(args.rename) <= len(args.outputs), (
'too few outputs to rename', args.rename, 'into', args.outputs)
- # Forbid changes to the loaded resource
- chmod(fetched_file, 0o444)
-
+ # Forbid changes to the loaded resource
+ chmod(fetched_file, 0o444)
+
if not os.path.isfile(fetched_file):
raise ResourceIsDirectoryError('Resource must be a file, not a directory: %s' % fetched_file)
@@ -332,16 +332,16 @@ def process(fetched_file, file_name, args, remove=True):
if args.untar_to:
ensure_dir(args.untar_to)
- # Extract only requested files
+ # Extract only requested files
try:
with tarfile.open(fetched_file, mode='r:*') as tar:
- inputs = set(map(os.path.normpath, args.rename + args.outputs[len(args.rename):]))
- members = [entry for entry in tar if os.path.normpath(os.path.join(args.untar_to, entry.name)) in inputs]
- tar.extractall(args.untar_to, members=members)
- # Forbid changes to the loaded resource data
- for root, _, files in os.walk(args.untar_to):
- for filename in files:
- chmod(os.path.join(root, filename), 0o444)
+ inputs = set(map(os.path.normpath, args.rename + args.outputs[len(args.rename):]))
+ members = [entry for entry in tar if os.path.normpath(os.path.join(args.untar_to, entry.name)) in inputs]
+ tar.extractall(args.untar_to, members=members)
+ # Forbid changes to the loaded resource data
+ for root, _, files in os.walk(args.untar_to):
+ for filename in files:
+ chmod(os.path.join(root, filename), 0o444)
except tarfile.ReadError as e:
logging.exception(e)
raise ResourceUnpackingError('File {} cannot be untared'.format(fetched_file))
@@ -354,12 +354,12 @@ def process(fetched_file, file_name, args, remove=True):
hardlink_or_copy(src, dst)
else:
logging.info('Renaming %s to %s', src, dst)
- if os.path.exists(dst):
- raise ResourceUnpackingError("Target file already exists ({} -> {})".format(src, dst))
+ if os.path.exists(dst):
+ raise ResourceUnpackingError("Target file already exists ({} -> {})".format(src, dst))
if remove:
rename_or_copy_and_remove(src, dst)
else:
- hardlink_or_copy(src, dst)
+ hardlink_or_copy(src, dst)
for path in args.outputs:
if not os.path.exists(path):
@@ -367,9 +367,9 @@ def process(fetched_file, file_name, args, remove=True):
if not os.path.isfile(path):
raise OutputIsDirectoryError('Output must be a file, not a directory: %s' % os.path.abspath(path))
if args.executable:
- chmod(path, os.stat(path).st_mode | 0o111)
+ chmod(path, os.stat(path).st_mode | 0o111)
if os.path.abspath(path) == os.path.abspath(fetched_file):
remove = False
-
+
if remove:
os.remove(fetched_file)