#!/usr/bin/env python import os import sys import platform import json URLS = ["https://storage.mds.yandex.net/get-devtools-opensource/471749/6a65891429890bff3fba00c421eb1911"] MD5 = "6a65891429890bff3fba00c421eb1911" RETRIES = 5 HASH_PREFIX = 10 HOME_DIR = os.path.expanduser('~') def create_dirs(path): try: os.makedirs(path) except OSError as e: import errno if e.errno != errno.EEXIST: raise return path def misc_root(): return create_dirs(os.getenv('YA_CACHE_DIR') or os.path.join(HOME_DIR, '.ya')) def tool_root(): return create_dirs(os.getenv('YA_CACHE_DIR_TOOLS') or os.path.join(misc_root(), 'tools')) def ya_token(): def get_token_from_file(): try: with open(os.environ.get('YA_TOKEN_PATH', os.path.join(HOME_DIR, '.ya_token')), 'r') as f: return f.read().strip() except: pass return os.getenv('YA_TOKEN') or get_token_from_file() TOOLS_DIR = tool_root() def uniq(size=6): import string import random return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(size)) def _fetch(url, into): import hashlib try: from urllib2 import urlopen from urllib2 import Request from urlparse import urlparse except ImportError: from urllib.request import urlopen from urllib.request import Request from urllib.parse import urlparse request = Request(str(url)) request.add_header('User-Agent', 'ya-bootstrap') if urlparse(url).netloc == 'proxy.sandbox.yandex-team.ru': token = ya_token() if token: request.add_header('Authorization', 'OAuth {}'.format(token)) md5 = hashlib.md5() sys.stderr.write('Downloading %s ' % url) sys.stderr.flush() conn = urlopen(request, timeout=10) sys.stderr.write('[') sys.stderr.flush() try: with open(into, 'wb') as f: while True: block = conn.read(1024 * 1024) sys.stderr.write('.') sys.stderr.flush() if block: md5.update(block) f.write(block) else: break return md5.hexdigest() finally: sys.stderr.write('] ') sys.stderr.flush() def _atomic_fetch(url, into, md5): tmp_dest = into + '.' + uniq() try: real_md5 = _fetch(url, tmp_dest) if real_md5 != md5: raise Exception('MD5 mismatched: %s differs from %s' % (real_md5, md5)) os.rename(tmp_dest, into) sys.stderr.write('OK\n') except Exception as e: sys.stderr.write('ERROR: ' + str(e) + '\n') raise finally: try: os.remove(tmp_dest) except OSError: pass def _extract(path, into): import tarfile tar = tarfile.open(path, errorlevel=2) tar.extractall(path=into) tar.close() def _get(urls, md5): dest_path = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX]) if not os.path.exists(dest_path): for iter in range(RETRIES): try: _atomic_fetch(urls[iter % len(urls)], dest_path, md5) break except Exception: if iter + 1 == RETRIES: raise else: import time time.sleep(iter) return dest_path def _get_dir(urls, md5, ya_name): dest_dir = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX] + '_d') if os.path.isfile(os.path.join(dest_dir, ya_name)): return dest_dir try: packed_path = _get(urls, md5) except Exception: if os.path.isfile(os.path.join(dest_dir, ya_name)): return dest_dir raise tmp_dir = dest_dir + '.' + uniq() try: try: _extract(packed_path, tmp_dir) except Exception: if os.path.isfile(os.path.join(dest_dir, ya_name)): return dest_dir raise try: os.rename(tmp_dir, dest_dir) except OSError as e: import errno if e.errno != errno.ENOTEMPTY: raise return dest_dir finally: import shutil shutil.rmtree(tmp_dir, ignore_errors=True) try: os.remove(packed_path) except Exception: pass def _mine_arc_root(): return os.path.dirname(os.path.realpath(__file__)) def main(): if not os.path.exists(TOOLS_DIR): os.makedirs(TOOLS_DIR) with open(_get(URLS, MD5), 'r') as fp: meta = json.load(fp)['data'] my_platform = platform.system().lower() my_machine = platform.machine().lower() if my_platform == 'linux': my_platform = 'linux-ppc64le' if 'ppc64le' in platform.platform() else 'linux_musl' if my_platform == 'darwin' and my_machine == 'arm64': my_platform = 'darwin-arm64' # match by max prefix length, prefer shortest best_key = max(meta.keys(), key=lambda x: (len(os.path.commonprefix([my_platform, x])), -len(x))) value = meta[best_key] if len(sys.argv) == 2 and sys.argv[1].startswith('--print-sandbox-id='): target = sys.argv[1].split('=')[1] best_target = max(meta.keys(), key=lambda x: len(os.path.commonprefix([target, x]))) sys.stdout.write(str(meta[best_target]['resource_id']) + '\n') exit(0) ya_name = {'win32': 'ya-bin.exe'}.get(best_key, 'ya-bin') # XXX ya_dir = _get_dir(value['urls'], value['md5'], ya_name) # Popen `args` must have `str` type ya_path = str(os.path.join(ya_dir, ya_name)) env = os.environ.copy() if 'YA_SOURCE_ROOT' not in env: src_root = _mine_arc_root() if src_root is not None: env['YA_SOURCE_ROOT'] = src_root # For more info see YT-14105 if 'LD_PRELOAD' in env: sys.stderr.write("Warn: LD_PRELOAD='{}' is specified and may affect the correct operation of the ya\n".format(env['LD_PRELOAD'])) if os.name == 'nt': import subprocess p = subprocess.Popen([ya_path] + sys.argv[1:], env=env) p.wait() sys.exit(p.returncode) else: os.execve(ya_path, [ya_path] + sys.argv[1:], env) if __name__ == '__main__': try: main() except Exception as e: sys.stderr.write('ERROR: ' + str(e) + '\n') sys.exit(1)