Canonization backend 1st part

Здесь мы добавляем в машинерию канонизации возможность подстановки в url, чтобы в будущем можно было загружать и использовать канонические данные из нескольких мест.

В этом ревью добавлена вся необходимая поддержка вокруг машинерии тестирования

В следующей итерации нужно покрыть тестами скачивание из  s3

3 files changed, 26 insertions, 17 deletions

diff --git a/build/plugins/lib/test_const/__init__.py b/build/plugins/lib/test_const/__init__.py
index 7157b7b8c5..125c622750 100644
--- a/build/plugins/lib/test_const/__init__.py
+++ b/build/plugins/lib/test_const/__init__.py
@@ -43,6 +43,15 @@ YT_RUN_TEST_TAR_NAME = "yt_run_test.tar"
 COVERAGE_CFLAGS = ["-fprofile-instr-generate", "-fcoverage-mapping", "-DCLANG_COVERAGE"]
 COVERAGE_LDFLAGS = ["-fprofile-instr-generate", "-fcoverage-mapping"]
 
+CANON_BACKEND_KEY = "canondata_backend"
+DEFAULT_CANONIZATION_BACKEND = "storage.yandex-team.ru/get-devtools"
+MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/'
+BACKEND_URI_PREFIX = 'https://{' + CANON_BACKEND_KEY + '}/'
+MDS_SCHEME = 'mds'
+CANON_MDS_RESOURCE_REGEX = re.compile(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)')
+CANON_BACKEND_RESOURCE_REGEX = re.compile(re.escape(BACKEND_URI_PREFIX) + r'(.*?)($|#)')
+CANON_SBR_RESOURCE_REGEX = re.compile(r'(sbr:/?/?(\d+))')
+
 MANDATORY_ENV_VAR_NAME = 'YA_MANDATORY_ENV_VARS'
 
 BUILD_FLAGS_ALLOWED_IN_CONTEXT = {
diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py
index b227fbb9fd..5087f5c532 100644
--- a/build/plugins/ytest.py
+++ b/build/plugins/ytest.py
@@ -20,14 +20,9 @@ import collections
 
 import ymake
 
-
-MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/'
-MDS_SCHEME = 'mds'
 CANON_DATA_DIR_NAME = 'canondata'
 CANON_OUTPUT_STORAGE = 'canondata_storage'
 CANON_RESULT_FILE_NAME = 'result.json'
-CANON_MDS_RESOURCE_REGEX = re.compile(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)')
-CANON_SBR_RESOURCE_REGEX = re.compile(r'(sbr:/?/?(\d+))')
 
 BLOCK_SEPARATOR = '============================================================='
 SPLIT_FACTOR_MAX_VALUE = 1000
@@ -1199,12 +1194,17 @@ def _load_canonical_file(filename, unit_path):
 
 
 def _get_resource_from_uri(uri):
-    m = CANON_MDS_RESOURCE_REGEX.match(uri)
+    m = consts.CANON_MDS_RESOURCE_REGEX.match(uri)
+    if m:
+        key = m.group(1)
+        return "{}:{}".format(consts.MDS_SCHEME, key)
+
+    m = consts.CANON_BACKEND_RESOURCE_REGEX.match(uri)
     if m:
-        res_id = m.group(1)
-        return "{}:{}".format(MDS_SCHEME, res_id)
+        key = m.group(1)
+        return "{}:{}".format(consts.MDS_SCHEME, key)
 
-    m = CANON_SBR_RESOURCE_REGEX.match(uri)
+    m = consts.CANON_SBR_RESOURCE_REGEX.match(uri)
     if m:
         # There might be conflict between resources, because all resources in sandbox have 'resource.tar.gz' name
         # That's why we use notation with '=' to specify specific path for resource
diff --git a/build/scripts/fetch_from_mds.py b/build/scripts/fetch_from_mds.py
index 5e4e656394..e9c838a7a8 100644
--- a/build/scripts/fetch_from_mds.py
+++ b/build/scripts/fetch_from_mds.py
@@ -5,32 +5,32 @@ import argparse
 
 import fetch_from
 
-MDS_PREFIX = "https://storage.yandex-team.ru/get-devtools/"
-
 
 def parse_args():
     parser = argparse.ArgumentParser()
     fetch_from.add_common_arguments(parser)
 
     parser.add_argument('--key', required=True)
+    parser.add_argument('--entrypoint', required=False, default="storage.yandex-team.ru/get-devtools")
+    parser.add_argument('--scheme', required=False, default="https")
 
     return parser.parse_args()
 
 
-def fetch(key):
-    parts = key.split("/")
+def fetch(args):
+    parts = args.key.split("/")
     if len(parts) != 3:
-        raise ValueError("Invalid MDS key '{}'".format(key))
+        raise ValueError("Invalid MDS key '{}'".format(args.key))
 
     _, sha1, file_name = parts
-
-    fetched_file = fetch_from.fetch_url(MDS_PREFIX + key, False, file_name, expected_sha1=sha1)
+    url = "{}://{}/{}".format(args.scheme, args.entrypoint, args.key)
+    fetched_file = fetch_from.fetch_url(url, False, file_name, expected_sha1=sha1)
 
     return fetched_file, file_name
 
 
 def main(args):
-    fetched_file, resource_file_name = fetch(args.key)
+    fetched_file, resource_file_name = fetch(args)
 
     fetch_from.process(fetched_file, resource_file_name, args)