1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
import os
import sys
import time
import logging
import argparse
import hashlib
import base64
import sky
import fetch_from
NPM_BASEURL = "http://npm.yandex-team.ru"
def parse_args():
parser = argparse.ArgumentParser()
fetch_from.add_common_arguments(parser)
parser.add_argument("--tarball-url", required=True)
parser.add_argument("--sky-id", required=True)
parser.add_argument("--integrity", required=True)
parser.add_argument("--integrity-algorithm", required=True)
return parser.parse_args()
def fetch(tarball_url, sky_id, integrity, integrity_algorithm, file_name, tries=5):
"""
:param name: package name
:type name: str
:param version: package version
:type version: str
:param sky_id: sky id of tarball
:type sky_id: str
:param integrity: tarball integrity (hex)
:type integrity: str
:param integrity_algorithm: integrity algorithm (known for openssl)
:type integrity_algorithm: str
:param tries: tries count
:type tries: int
:return: path to fetched file
:rtype: str
"""
# if sky.is_avaliable() and 'NOTS_FETCH_FROM_HTTP_ONLY' not in os.environ:
# fetcher = lambda: sky.fetch(sky_id, file_name)
# else:
# Отключаем походы через скай
# TODO: https://st.yandex-team.ru/FBP-351
if 'NOTS_FETCH_FROM_SKY' in os.environ and sky.is_avaliable():
fetcher = lambda: sky.fetch(sky_id, file_name)
else:
fetcher = lambda: _fetch_via_http(tarball_url, integrity, integrity_algorithm, file_name)
fetched_file = None
exc_info = None
for i in range(0, tries):
try:
fetched_file = fetcher()
exc_info = None
break
except Exception as e:
logging.exception(e)
exc_info = exc_info or sys.exc_info()
time.sleep(i)
if exc_info:
raise exc_info[0], exc_info[1], exc_info[2]
return fetched_file
def _fetch_via_http(tarball_url, integrity, integrity_algorithm, file_name):
is_abs_url = tarball_url.startswith("https://") or tarball_url.startswith("http://")
url_delim = "" if tarball_url.startswith("/") else "/"
url = tarball_url if is_abs_url else NPM_BASEURL + url_delim + tarball_url
hashobj = hashlib.new(integrity_algorithm)
fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update])
checksum = base64.b64encode(hashobj.digest()).decode('utf-8')
if checksum != integrity:
raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format(
integrity,
checksum,
file_name,
))
return fetched_file
def main(args):
file_name = os.path.basename(args.copy_to)
fetched_file = fetch(args.tarball_url, args.sky_id, args.integrity, args.integrity_algorithm, file_name)
fetch_from.process(fetched_file, file_name, args)
if __name__ == "__main__":
args = parse_args()
fetch_from.setup_logging(args, os.path.basename(__file__))
try:
main(args)
except Exception as e:
logging.exception(e)
print >>sys.stderr, open(args.abs_log_path).read()
sys.stderr.flush()
import error
sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1)
|
