intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

1 files changed, 239 insertions, 0 deletions

diff --git a/ya b/ya
new file mode 100755
index 0000000000..2dd08820a0
--- /dev/null
+++ b/ya
@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+import os
+import sys
+import platform
+import json
+
+URLS = [u'https://proxy.sandbox.yandex-team.ru/2765019439', 'https://storage.yandex-team.ru/get-devtools/1777230/2effc2099f6739ec124c100379cd6c2fae12a11c/tmpK85OCD']
+MD5 = 'dd2488d08eab0e3839b27f65f4b78c51'
+
+RETRIES = 5
+HASH_PREFIX = 10
+
+HOME_DIR = os.path.expanduser('~')
+
+
+def create_dirs(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        import errno
+
+        if e.errno != errno.EEXIST:
+            raise
+
+    return path
+
+
+def misc_root():
+    return create_dirs(os.getenv('YA_CACHE_DIR') or os.path.join(HOME_DIR, '.ya'))
+
+
+def tool_root():
+    return create_dirs(os.getenv('YA_CACHE_DIR_TOOLS') or os.path.join(misc_root(), 'tools'))
+
+
+def ya_token():
+    def get_token_from_file():
+        try:
+            with open(os.environ.get('YA_TOKEN_PATH', os.path.join(HOME_DIR, '.ya_token')), 'r') as f:
+                return f.read().strip()
+        except:
+            pass
+    return os.getenv('YA_TOKEN') or get_token_from_file()
+
+
+TOOLS_DIR = tool_root()
+
+
+def uniq(size=6):
+    import string
+    import random
+
+    return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(size))
+
+
+def _fetch(url, into):
+    import hashlib
+
+    try:
+        from urllib2 import urlopen
+        from urllib2 import Request
+        from urlparse import urlparse
+    except ImportError:
+        from urllib.request import urlopen
+        from urllib.request import Request
+        from urllib.parse import urlparse
+
+    request = Request(str(url))
+    request.add_header('User-Agent', 'ya-bootstrap')
+    if urlparse(url).netloc == 'proxy.sandbox.yandex-team.ru':
+        token = ya_token()
+        if token:
+            request.add_header('Authorization', 'OAuth {}'.format(token))
+
+    md5 = hashlib.md5()
+    sys.stderr.write('Downloading %s ' % url)
+    sys.stderr.flush()
+    conn = urlopen(request, timeout=10)
+    sys.stderr.write('[')
+    sys.stderr.flush()
+    try:
+        with open(into, 'wb') as f:
+            while True:
+                block = conn.read(1024 * 1024)
+                sys.stderr.write('.')
+                sys.stderr.flush()
+                if block:
+                    md5.update(block)
+                    f.write(block)
+                else:
+                    break
+        return md5.hexdigest()
+
+    finally:
+        sys.stderr.write('] ')
+        sys.stderr.flush()
+
+
+def _atomic_fetch(url, into, md5):
+    tmp_dest = into + '.' + uniq()
+    try:
+        real_md5 = _fetch(url, tmp_dest)
+        if real_md5 != md5:
+            raise Exception('MD5 mismatched: %s differs from %s' % (real_md5, md5))
+        os.rename(tmp_dest, into)
+        sys.stderr.write('OK\n')
+    except Exception as e:
+        sys.stderr.write('ERROR: ' + str(e) + '\n')
+        raise
+    finally:
+        try:
+            os.remove(tmp_dest)
+        except OSError:
+            pass
+
+
+def _extract(path, into):
+    import tarfile
+
+    tar = tarfile.open(path, errorlevel=2)
+    tar.extractall(path=into)
+    tar.close()
+
+
+def _get(urls, md5):
+    dest_path = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX])
+
+    if not os.path.exists(dest_path):
+        for iter in range(RETRIES):
+            try:
+                _atomic_fetch(urls[iter % len(urls)], dest_path, md5)
+                break
+            except Exception:
+                if iter + 1 == RETRIES:
+                    raise
+                else:
+                    import time
+                    time.sleep(iter)
+
+    return dest_path
+
+
+def _get_dir(urls, md5, ya_name):
+    dest_dir = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX] + '_d')
+
+    if os.path.isfile(os.path.join(dest_dir, ya_name)):
+        return dest_dir
+
+    try:
+        packed_path = _get(urls, md5)
+    except Exception:
+        if os.path.isfile(os.path.join(dest_dir, ya_name)):
+            return dest_dir
+        raise
+
+    tmp_dir = dest_dir + '.' + uniq()
+    try:
+        try:
+            _extract(packed_path, tmp_dir)
+        except Exception:
+            if os.path.isfile(os.path.join(dest_dir, ya_name)):
+                return dest_dir
+            raise
+
+        try:
+            os.rename(tmp_dir, dest_dir)
+        except OSError as e:
+            import errno
+            if e.errno != errno.ENOTEMPTY:
+                raise
+
+        return dest_dir
+    finally:
+        import shutil
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+        try:
+            os.remove(packed_path)
+        except Exception:
+            pass
+
+
+def _mine_arc_root():
+    return os.path.dirname(os.path.realpath(__file__))
+
+
+def main():
+    if not os.path.exists(TOOLS_DIR):
+        os.makedirs(TOOLS_DIR)
+
+    with open(_get(URLS, MD5), 'r') as fp:
+        meta = json.load(fp)['data']
+    my_platform = platform.system().lower()
+    my_machine = platform.machine().lower()
+    if my_platform == 'linux':
+        my_platform = 'linux-ppc64le' if 'ppc64le' in platform.platform() else 'linux_musl'
+    if my_platform == 'darwin' and my_machine == 'arm64':
+        my_platform = 'darwin-arm64'
+
+    # match by max prefix length, prefer shortest
+    best_key = max(meta.keys(), key=lambda x: (len(os.path.commonprefix([my_platform, x])), -len(x)))
+    value = meta[best_key]
+
+    if len(sys.argv) == 2 and sys.argv[1].startswith('--print-sandbox-id='):
+        target = sys.argv[1].split('=')[1]
+        best_target = max(meta.keys(), key=lambda x: len(os.path.commonprefix([target, x])))
+        sys.stdout.write(str(meta[best_target]['resource_id']) + '\n')
+        exit(0)
+
+    ya_name = {'win32': 'ya-bin.exe'}.get(best_key, 'ya-bin')  # XXX
+    ya_dir = _get_dir(value['urls'], value['md5'], ya_name)
+
+    # Popen `args` must have `str` type
+    ya_path = str(os.path.join(ya_dir, ya_name))
+
+    env = os.environ.copy()
+    if 'YA_SOURCE_ROOT' not in env:
+        src_root = _mine_arc_root()
+        if src_root is not None:
+            env['YA_SOURCE_ROOT'] = src_root
+    # For more info see YT-14105
+    if 'LD_PRELOAD' in env:
+        sys.stderr.write("Warn: LD_PRELOAD='{}' is specified and may affect the correct operation of the ya\n".format(env['LD_PRELOAD']))
+
+    if os.name == 'nt':
+        import subprocess
+
+        p = subprocess.Popen([ya_path] + sys.argv[1:], env=env)
+        p.wait()
+        sys.exit(p.returncode)
+    else:
+        os.execve(ya_path, [ya_path] + sys.argv[1:], env)
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as e:
+        sys.stderr.write('ERROR: ' + str(e) + '\n')
+        sys.exit(1)