[devtools/contrib/piglet/projects/ydb/config.yaml] Setup opensource version of ya make in ydb's github repo

5 files changed, 584 insertions, 1 deletions

diff --git a/.mapping.json b/.mapping.json
index 45b8a0ef34..5e2660d705 100644
--- a/.mapping.json
+++ b/.mapping.json
@@ -43,9 +43,13 @@
   "ROADMAP.md":"ydb/github_toplevel/ROADMAP.md",
   "SECURITY.md":"ydb/github_toplevel/SECURITY.md",
   "build/internal/conf/internal.conf":"ydb/github_toplevel/build/internal/conf/internal.conf",
+  "build/mapping.conf.json":"devtools/ya/opensource/mapping.conf.json",
+  "build/ya.conf.json":"devtools/ya/opensource/ya.conf.json",
   "clang.toolchain":"ydb/github_toplevel/clang.toolchain",
   "generate_cmake":"ydb/github_toplevel/generate_cmake",
   "go.mod":"ydb/github_toplevel/go.mod",
   "go.sum":"ydb/github_toplevel/go.sum",
-  "scripts/generate_dump.sh":"ydb/github_toplevel/scripts/generate_dump.sh"
+  "scripts/generate_dump.sh":"ydb/github_toplevel/scripts/generate_dump.sh",
+  "ya":"devtools/ya/opensource/ya",
+  "ya.conf":"devtools/ya/opensource/ya.conf"
 }
\ No newline at end of file
diff --git a/build/mapping.conf.json b/build/mapping.conf.json
new file mode 100644
index 0000000000..52e8b47404
--- /dev/null
+++ b/build/mapping.conf.json
@@ -0,0 +1,134 @@
+{
+    "bottles": {},
+    "resources": {
+        "4748237118": "http://s3.mds.yandex.net/test-devtools-registry/4748237118",
+        "4748241015": "http://s3.mds.yandex.net/test-devtools-registry/4748241015",
+        "4748280193": "http://s3.mds.yandex.net/test-devtools-registry/4748280193",
+        "4748318935": "http://s3.mds.yandex.net/test-devtools-registry/4748318935",
+        "4748341848": "http://s3.mds.yandex.net/test-devtools-registry/4748341848",
+        "2319130389": "http://s3.mds.yandex.net/test-devtools-registry/2319130389",
+        "1966560555": "http://s3.mds.yandex.net/test-devtools-registry/1966560555",
+        "360916612": "http://s3.mds.yandex.net/test-devtools-registry/360916612",
+        "2989597929": "http://s3.mds.yandex.net/test-devtools-registry/2989597929",
+        "2989596911": "http://s3.mds.yandex.net/test-devtools-registry/2989596911",
+        "2989598506": "http://s3.mds.yandex.net/test-devtools-registry/2989598506",
+        "3864777619": "http://s3.mds.yandex.net/test-devtools-registry/3864777619",
+        "2989596149": "http://s3.mds.yandex.net/test-devtools-registry/2989596149",
+        "2989597467": "http://s3.mds.yandex.net/test-devtools-registry/2989597467",
+        "3877389803": "http://s3.mds.yandex.net/test-devtools-registry/3877389803",
+        "3877389564": "http://s3.mds.yandex.net/test-devtools-registry/3877389564",
+        "3877390008": "http://s3.mds.yandex.net/test-devtools-registry/3877390008",
+        "3877389037": "http://s3.mds.yandex.net/test-devtools-registry/3877389037",
+        "3877389242": "http://s3.mds.yandex.net/test-devtools-registry/3877389242",
+        "3877388826": "http://s3.mds.yandex.net/test-devtools-registry/3877388826",
+        "5011256055": "http://s3.mds.yandex.net/test-devtools-registry/5011256055",
+        "5011278671": "http://s3.mds.yandex.net/test-devtools-registry/5011278671",
+        "3833498694": "http://s3.mds.yandex.net/test-devtools-registry/3833498694",
+        "4940640829": "http://s3.mds.yandex.net/test-devtools-registry/4940640829",
+        "4940633745": "http://s3.mds.yandex.net/test-devtools-registry/4940633745",
+        "4588378868": "http://s3.mds.yandex.net/test-devtools-registry/4588378868",
+        "4588370003": "http://s3.mds.yandex.net/test-devtools-registry/4588370003",
+        "4588389789": "http://s3.mds.yandex.net/test-devtools-registry/4588389789",
+        "4588396480": "http://s3.mds.yandex.net/test-devtools-registry/4588396480",
+        "4635443430": "http://s3.mds.yandex.net/test-devtools-registry/4635443430",
+        "4635426988": "http://s3.mds.yandex.net/test-devtools-registry/4635426988",
+        "4635449932": "http://s3.mds.yandex.net/test-devtools-registry/4635449932",
+        "4635451669": "http://s3.mds.yandex.net/test-devtools-registry/4635451669",
+        "4991298338": "http://s3.mds.yandex.net/test-devtools-registry/4991298338",
+        "4991308971": "http://s3.mds.yandex.net/test-devtools-registry/4991308971",
+        "4991323673": "http://s3.mds.yandex.net/test-devtools-registry/4991323673",
+        "4991310596": "http://s3.mds.yandex.net/test-devtools-registry/4991310596",
+        "4991328977": "http://s3.mds.yandex.net/test-devtools-registry/4991328977"
+    },
+    "resources_descriptions": {
+        "4748237118": "Clang 14.0.6 for darwin arm64",
+        "4748241015": "Clang 14.0.6 for darwin x86_64",
+        "4748280193": "Clang 14.0.6 for linux aarch64",
+        "4748318935": "Clang 14.0.6 for linux x86_64",
+        "4748341848": "Clang 14.0.6 for windows-x86_64",
+        "2319130389": "GDB 8.3 toolkit for darwin-arm64",
+        "1966560555": "Ubuntu 14 x86-64 native SDK (patched, v3)",
+        "360916612": "binutils 2.26 for linux_ubuntu_10.04_lucid",
+        "2989597929": "devtools/huge_python/python for darwin",
+        "2989596911": "devtools/huge_python/python for darwin-arm64",
+        "2989598506": "devtools/huge_python/python for linux",
+        "3864777619": "devtools/huge_python/python for linux-aarch64",
+        "2989596149": "devtools/huge_python/python for linux-ppc64le",
+        "2989597467": "devtools/huge_python/python for win32",
+        "3877389803": "devtools/huge_python3/python3 for darwin",
+        "3877389564": "devtools/huge_python3/python3 for darwin-arm64",
+        "3877390008": "devtools/huge_python3/python3 for linux",
+        "3877389037": "devtools/huge_python3/python3 for linux-aarch64",
+        "3877389242": "devtools/huge_python3/python3 for linux-ppc64le",
+        "3877388826": "devtools/huge_python3/python3 for win32-clang-cl",
+        "5011256055": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
+        "5011278671": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
+        "3833498694": "gdb 11.2 for osx_10.15_catalina",
+        "4940640829": "gdb for linux-aarch64",
+        "4940633745": "gdb for linux-x86_64",
+        "4588378868": "lld darwin x86_64",
+        "4588370003": "lld for darwin arm64",
+        "4588389789": "lld for linux aarch64",
+        "4588396480": "lld for linux x86_64",
+        "4635443430": "ya-tc for darwin",
+        "4635426988": "ya-tc for darwin-arm64",
+        "4635449932": "ya-tc for linux",
+        "4635451669": "ya-tc for linux-aarch64",
+        "4991298338": "ymake for darwin",
+        "4991308971": "ymake for darwin-arm64",
+        "4991323673": "ymake for linux",
+        "4991310596": "ymake for linux-aarch64",
+        "4991328977": "ymake.exe for win32-clang-cl"
+    },
+    "tasks": {
+        "1032896450": {
+            "darwin-arm64": "2319130389"
+        },
+        "1549947872": {
+            "linux-aarch64": "3864777619"
+        },
+        "1277485811": {
+            "darwin": "2989597929",
+            "darwin-arm64": "2989596911",
+            "linux": "2989598506",
+            "linux-ppc64le": "2989596149",
+            "win32": "2989597467"
+        },
+        "1539939628": {
+            "Darwin-19.6.0-x86_64-i386-64bit": "3833498694"
+        },
+        "1885745585": {
+            "linux-aarch64": "4940640829"
+        },
+        "1885743377": {
+            "linux-x86_64": "4940633745"
+        },
+        "1797086833": {
+            "darwin": "4635443430"
+        },
+        "1797086849": {
+            "darwin-arm64": "4635426988"
+        },
+        "1797086885": {
+            "linux": "4635449932"
+        },
+        "1797086903": {
+            "linux-aarch64": "4635451669"
+        },
+        "1901124324": {
+            "darwin": "4991298338"
+        },
+        "1901124336": {
+            "darwin-arm64": "4991308971"
+        },
+        "1901124267": {
+            "linux": "4991323673"
+        },
+        "1901124396": {
+            "linux-aarch64": "4991310596"
+        },
+        "1901124302": {
+            "win32-clang-cl": "4991328977"
+        }
+    }
+}
\ No newline at end of file
diff --git a/build/ya.conf.json b/build/ya.conf.json
new file mode 100644
index 0000000000..0967ef424b
--- /dev/null
+++ b/build/ya.conf.json
@@ -0,0 +1 @@
+{}
diff --git a/ya b/ya
new file mode 100755
index 0000000000..813ae7119b
--- /dev/null
+++ b/ya
@@ -0,0 +1,349 @@
+#!/usr/bin/env python
+
+# Please, keep this script in sync with arcadia/ya
+
+import os
+import sys
+import platform
+
+RETRIES = 5
+HASH_PREFIX = 10
+
+REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://s3.mds.yandex.net/devtools-registry")
+
+
+PLATFORM_MAP = {
+    "data": {
+        "darwin": {
+            "md5": "1356748b6bb7ce19359bc1c87d81cf5e",
+            "urls": [
+                f"{REGISTRY_ENDPOINT}/4948880410",
+            ],
+        },
+        "darwin-arm64": {
+            "md5": "2ba999e77660102f78c2584931eef3a2",
+            "urls": [
+                f"{REGISTRY_ENDPOINT}/4948879923",
+            ],
+        },
+        "linux-aarch64": {
+            "md5": "731df682969dddc89bd7f3160960ffa1",
+            "urls": [
+                f"{REGISTRY_ENDPOINT}/4948879460",
+            ],
+        },
+        "win32-clang-cl": {
+            "md5": "e2289cac60bf800470d858af48fce490",
+            "urls": [
+                f"{REGISTRY_ENDPOINT}/4948880870",
+            ],
+        },
+        "linux": {
+            "md5": "c71678ed371ae966d7c0235352a398e3",
+            "urls": [
+                f"{REGISTRY_ENDPOINT}/4948881174",
+            ],
+        },
+    }
+}
+# End of mapping
+
+
+def create_dirs(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        import errno
+
+        if e.errno != errno.EEXIST:
+            raise
+
+    return path
+
+
+def home_dir():
+    # Do not trust $HOME, as it is unreliable in certain environments
+    # Temporarily delete os.environ["HOME"] to force reading current home directory from /etc/passwd
+    home_from_env = os.environ.pop("HOME", None)
+    try:
+        home_from_passwd = os.path.expanduser("~")
+        if os.path.isabs(home_from_passwd):
+            # This home dir is valid, prefer it over $HOME
+            return home_from_passwd
+        else:
+            # When python is built with musl (this is quire weird though),
+            # only users from /etc/passwd will be properly resolved,
+            # as musl does not have nss module for LDAP integration.
+            return home_from_env
+
+    finally:
+        if home_from_env is not None:
+            os.environ["HOME"] = home_from_env
+
+
+def misc_root():
+    return create_dirs(os.getenv('YA_CACHE_DIR') or os.path.join(home_dir(), '.ya'))
+
+
+def tool_root():
+    return create_dirs(os.getenv('YA_CACHE_DIR_TOOLS') or os.path.join(misc_root(), 'tools'))
+
+
+# TODO: remove when switched to S3, won't be needed in OSS
+def ya_token():
+    def get_token_from_file():
+        try:
+            with open(os.environ.get('YA_TOKEN_PATH', os.path.join(home_dir(), '.ya_token')), 'r') as f:
+                return f.read().strip()
+        except:
+            pass
+
+    return os.getenv('YA_TOKEN') or get_token_from_file()
+
+
+TOOLS_DIR = tool_root()
+
+
+def uniq(size=6):
+    import string
+    import random
+
+    return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(size))
+
+
+_ssl_is_tuned = False
+
+
+def _tune_ssl():
+    global _ssl_is_tuned
+    if _ssl_is_tuned:
+        return
+
+    try:
+        import ssl
+
+        ssl._create_default_https_context = ssl._create_unverified_context
+    except AttributeError:
+        pass
+
+    try:
+        import urllib3
+
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+    except (AttributeError, ImportError):
+        pass
+    _ssl_is_tuned = True
+
+
+def _fetch(url, into):
+    import hashlib
+
+    _tune_ssl()
+
+    from urllib.request import urlopen
+    from urllib.request import Request
+    from urllib.parse import urlparse
+
+    request = Request(str(url))
+    # TODO: Remove when switched to S3 distribution
+    request.add_header('User-Agent', 'ya-bootstrap')
+    token = ya_token()
+    if token:
+        request.add_header('Authorization', 'OAuth {}'.format(token))
+
+    md5 = hashlib.md5()
+    sys.stderr.write('Downloading %s ' % url)
+    sys.stderr.flush()
+    conn = urlopen(request, timeout=10)
+    sys.stderr.write('[')
+    sys.stderr.flush()
+    try:
+        with open(into, 'wb') as f:
+            while True:
+                block = conn.read(1024 * 1024)
+                sys.stderr.write('.')
+                sys.stderr.flush()
+                if block:
+                    md5.update(block)
+                    f.write(block)
+                else:
+                    break
+        return md5.hexdigest()
+
+    finally:
+        sys.stderr.write('] ')
+        sys.stderr.flush()
+
+
+def _atomic_fetch(url, into, md5):
+    tmp_dest = into + '.' + uniq()
+    try:
+        real_md5 = _fetch(url, tmp_dest)
+        if real_md5 != md5:
+            raise Exception('MD5 mismatched: %s differs from %s' % (real_md5, md5))
+        os.rename(tmp_dest, into)
+        sys.stderr.write('OK\n')
+    except Exception as e:
+        sys.stderr.write('ERROR: ' + str(e) + '\n')
+        raise
+    finally:
+        try:
+            os.remove(tmp_dest)
+        except OSError:
+            pass
+
+
+def _extract(path, into):
+    import tarfile
+
+    tar = tarfile.open(path, errorlevel=2)
+
+    # tar.extractall() will try to set file ownership according to the attributes stored in the archive
+    # by calling TarFile.chown() method.
+    # As this information is hardly relevant to the point of deployment / extraction,
+    # it will just fail (python2) if ya is executed with root euid, or silently set non-existent numeric owner (python3)
+    # to the files being extracted.
+    # mock it with noop to retain current user ownership.
+    tar.chown = lambda *args, **kwargs: None
+
+    tar.extractall(path=into)
+    tar.close()
+
+
+def _get(urls, md5):
+    dest_path = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX])
+
+    if not os.path.exists(dest_path):
+        for iter in range(RETRIES):
+            try:
+                _atomic_fetch(urls[iter % len(urls)], dest_path, md5)
+                break
+            except Exception:
+                if iter + 1 == RETRIES:
+                    raise
+                else:
+                    import time
+
+                    time.sleep(iter)
+
+    return dest_path
+
+
+def _get_dir(urls, md5, ya_name):
+    dest_dir = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX] + '_d')
+
+    if os.path.isfile(os.path.join(dest_dir, ya_name)):
+        return dest_dir
+
+    try:
+        packed_path = _get(urls, md5)
+    except Exception:
+        if os.path.isfile(os.path.join(dest_dir, ya_name)):
+            return dest_dir
+        raise
+
+    tmp_dir = dest_dir + '.' + uniq()
+    try:
+        try:
+            _extract(packed_path, tmp_dir)
+        except Exception:
+            if os.path.isfile(os.path.join(dest_dir, ya_name)):
+                return dest_dir
+            raise
+
+        try:
+            os.rename(tmp_dir, dest_dir)
+        except OSError as e:
+            import errno
+
+            if e.errno != errno.ENOTEMPTY:
+                raise
+
+        return dest_dir
+    finally:
+        import shutil
+
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+        try:
+            os.remove(packed_path)
+        except Exception:
+            pass
+
+
+def _mine_repo_root():
+    # We think that this script is located in the root of the repo.
+    return os.path.dirname(os.path.realpath(__file__))
+
+
+def main():
+    if not os.path.exists(TOOLS_DIR):
+        os.makedirs(TOOLS_DIR)
+
+    result_args = sys.argv[1:]
+
+    meta = PLATFORM_MAP['data']
+    my_platform = platform.system().lower()
+    my_machine = platform.machine().lower()
+    if my_platform == 'linux':
+        if 'ppc64le' in platform.platform():
+            my_platform = 'linux-ppc64le'
+        elif 'aarch64' in platform.platform():
+            my_platform = 'linux-aarch64'
+        else:
+            my_platform = 'linux_musl'
+    if my_platform == 'darwin' and my_machine == 'arm64':
+        my_platform = 'darwin-arm64'
+
+    def _platform_key(target_platform):
+        """match by max prefix length, prefer shortest"""
+
+        def _key_for_platform(platform):
+            return len(os.path.commonprefix([target_platform, platform])), -len(platform)
+
+        return _key_for_platform
+
+    best_key = max(meta.keys(), key=_platform_key(my_platform))
+    value = meta[best_key]
+
+    ya_name = {'win32': 'ya-bin.exe', 'win32-clang-cl': 'ya-bin.exe'}.get(best_key, 'ya-bin')  # XXX
+    ya_dir = _get_dir(value['urls'], value['md5'], ya_name)
+
+    # Popen `args` must have `str` type
+    ya_path = str(os.path.join(ya_dir, ya_name))
+
+    env = os.environ.copy()
+    if 'YA_SOURCE_ROOT' not in env:
+        src_root = _mine_repo_root()
+        if src_root is not None:
+            env['YA_SOURCE_ROOT'] = src_root
+
+    for env_name in [
+        'LD_PRELOAD',
+        'Y_PYTHON_SOURCE_ROOT',
+    ]:
+        if env_name in os.environ:
+            sys.stderr.write(
+                "Warn: {}='{}' is specified and may affect the correct operation of the ya\n".format(
+                    env_name, env[env_name]
+                )
+            )
+
+    if os.name == 'nt':
+        import subprocess
+
+        p = subprocess.Popen([ya_path] + result_args, env=env)
+        p.wait()
+        sys.exit(p.returncode)
+    else:
+        os.execve(ya_path, [ya_path] + result_args, env)
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as e:
+        sys.stderr.write('ERROR: ' + str(e) + '\n')
+        from traceback import format_exc
+
+        sys.stderr.write(format_exc() + "\n")
+        sys.exit(1)
diff --git a/ya.conf b/ya.conf
new file mode 100644
index 0000000000..fceed51d64
--- /dev/null
+++ b/ya.conf
@@ -0,0 +1,95 @@
+# Please keep this in sync with arcadia/ya.conf
+
+build_cache = true
+build_cache_conf = ['cas_logging=true', 'graph_info=true']
+build_cache_master = true
+cache_codec = ''
+cache_size = 150374182400
+content_uids = true
+dir_outputs = true
+dir_outputs_test_mode = true
+dump_debug_enabled = true
+fail_maven_export_with_tests = true
+incremental_build_dirs_cleanup = true
+oauth_exchange_ssh_keys = true
+remove_implicit_data_path = true
+remove_result_node = true
+tools_cache = true
+tools_cache_master = true
+use_atd_revisions_info = true
+use_jstyle_server = true
+use_command_file_in_testtool = true
+
+[test_tool3_handlers]
+build_clang_coverage_report = true
+build_go_coverage_report = true
+build_python_coverage_report = false
+build_sancov_coverage_report = true
+build_ts_coverage_report = true
+canonization_result_node = true
+canonize = false
+check_external = false
+check_mds = false
+check_resource = false
+checkout = false
+cov_merge_vfs = false
+create_allure_report = true
+download = true
+list_result_node = true
+list_tests = false
+merge_coverage_inplace = true
+merge_python_coverage = true
+minimize_fuzz_corpus = true
+populate_token_to_sandbox_vault = true
+resolve_clang_coverage = true
+resolve_go_coverage = true
+resolve_java_coverage = true
+resolve_python_coverage = false
+resolve_sancov_coverage = true
+resolve_ts_coverage = true
+result_node = true
+results_accumulator = false
+results_merger = false
+run_boost_test = true
+run_check = false
+run_clang_tidy = false
+run_classpath_clash = true
+run_coverage_extractor = true
+run_custom_lint = true
+run_diff_test = false
+run_eslint = false
+run_exectest = false
+run_fuzz = false
+run_fuzz_result_node = true
+run_g_benchmark = true
+run_go_fmt = false
+run_go_test = false
+run_go_vet = true
+run_gtest = true
+run_hermione = true
+run_hermione_beta = true
+run_javastyle = false
+run_jest = true
+run_ktlint_test = false
+run_pyimports = true
+run_skipped_test = true
+run_test = false
+run_ut = false
+run_y_benchmark = true
+sandbox_run_test = true
+unify_clang_coverage = true
+upload = false
+upload_coverage = false
+ytexec_run_test = false
+
+# ===== opensource only table params =====
+
+[host_platform_flags]
+OPENSOURCE = "yes"
+USE_LOCAL_SWIG = "yes"
+USE_PREBUILT_TOOLS = "no"
+
+[flags]
+OPENSOURCE = "yes"
+USE_LOCAL_SWIG = "yes"
+USE_PREBUILT_TOOLS = "no"