Docker image fetch macro

commit_hash:0899d0942e1f7ad8dd12e28f6b3f088982660699

5 files changed, 80 insertions, 0 deletions

diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py
index 07ff9ce0f2..51349aa4ae 100644
--- a/build/plugins/_dart_fields.py
+++ b/build/plugins/_dart_fields.py
@@ -926,6 +926,35 @@ class TestData:
         return {cls.KEY: serialize_list(get_values_list(unit, "TEST_DATA_VALUE"))}
 
 
+class DockerImage:
+    KEY = 'DOCKER-IMAGES'
+
+    @staticmethod
+    def _validate(images):
+        docker_image_re = consts.DOCKER_LINK_RE
+        for img in images:
+            msg = None
+            if "=" in img:
+                link, _ = img.rsplit('=', 1)
+                if docker_image_re.match(link) is None:
+                    msg = 'Invalid docker url format: {}. Link should be provided in format docker://<repo>@sha256:<digest>'.format(
+                        link
+                    )
+            else:
+                msg = 'Invalid docker image: {}. Image should be provided in format <link>=<tag>'.format(img)
+            if msg:
+                ymake.report_configure_error(msg)
+                raise DartValueError(msg)
+
+    @classmethod
+    def value(cls, unit, flat_args, spec_args):
+        raw_value = get_values_list(unit, 'DOCKER_IMAGES_VALUE')
+        images = sorted(raw_value)
+        if images:
+            cls._validate(images)
+        return {cls.KEY: serialize_list(images)}
+
+
 class TsConfigPath:
     KEY = 'TS_CONFIG_PATH'
 
diff --git a/build/plugins/lib/test_const/__init__.py b/build/plugins/lib/test_const/__init__.py
index 6086251f52..c6f16bdb09 100644
--- a/build/plugins/lib/test_const/__init__.py
+++ b/build/plugins/lib/test_const/__init__.py
@@ -211,6 +211,8 @@ WINE32_TOOL = 'WINE32_TOOL_RESOURCE_GLOBAL'
 
 DEFAULT_CRASHED_STATUS_COMMENT = "Test crashed"
 
+DOCKER_LINK_RE = re.compile(r"(docker:\/\/)(\S+?)(\/\S*)?\@sha\d+:(\w+)")
+
 
 class Enum(object):
     @classmethod
diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py
index 46ee8d2599..6b1a77e235 100644
--- a/build/plugins/ytest.py
+++ b/build/plugins/ytest.py
@@ -772,6 +772,7 @@ def onadd_check_py_imports(fields, unit, *args):
         df.ModuleLang.value,
         df.BinaryPath.stripped,
         df.TestRunnerBin.value,
+        df.DockerImage.value,
     )
 )
 def onadd_pytest_bin(fields, unit, *args):
@@ -831,6 +832,7 @@ def onadd_pytest_bin(fields, unit, *args):
         df.TestClasspathOrigins.value,
         df.TestClasspathDeps.value,
         df.TestJar.value,
+        df.DockerImage.value,
     )
 )
 def onjava_test(fields, unit, *args):
@@ -911,6 +913,7 @@ def onrun(unit, *args):
         df.TestName.filename_without_pkg_ext,
         df.TestedProjectName.path_filename_basename_without_pkg_ext,
         df.BinaryPath.stripped_without_pkg_ext,
+        df.DockerImage.value,
     )
 )
 def onsetup_exectest(fields, unit, *args):
@@ -1144,6 +1147,7 @@ def clang_tidy(fields, unit, *args):
         df.Requirements.from_macro_args_and_unit,
         df.TestPartition.value,
         df.ModuleLang.value,
+        df.DockerImage.value,
     )
 )
 def unittest_py(fields, unit, *args):
@@ -1177,6 +1181,7 @@ def unittest_py(fields, unit, *args):
         df.Requirements.from_macro_args_and_unit,
         df.TestPartition.value,
         df.ModuleLang.value,
+        df.DockerImage.value,
     )
 )
 def gunittest(fields, unit, *args):
@@ -1211,6 +1216,7 @@ def gunittest(fields, unit, *args):
         df.TestPartition.value,
         df.ModuleLang.value,
         df.BenchmarkOpts.value,
+        df.DockerImage.value,
     )
 )
 def g_benchmark(fields, unit, *args):
@@ -1244,6 +1250,7 @@ def g_benchmark(fields, unit, *args):
         df.Requirements.from_macro_args_and_unit,
         df.TestPartition.value,
         df.ModuleLang.value,
+        df.DockerImage.value,
     )
 )
 def go_test(fields, unit, *args):
@@ -1277,6 +1284,7 @@ def go_test(fields, unit, *args):
         df.TestData.from_macro_args_and_unit,
         df.Requirements.from_macro_args_and_unit,
         df.TestPartition.value,
+        df.DockerImage.value,
     )
 )
 def boost_test(fields, unit, *args):
@@ -1312,6 +1320,7 @@ def boost_test(fields, unit, *args):
         df.FuzzDicts.value,
         df.FuzzOpts.value,
         df.Fuzzing.value,
+        df.DockerImage.value,
     )
 )
 def fuzz_test(fields, unit, *args):
@@ -1348,6 +1357,7 @@ def fuzz_test(fields, unit, *args):
         df.TestPartition.value,
         df.ModuleLang.value,
         df.BenchmarkOpts.value,
+        df.DockerImage.value,
     )
 )
 def y_benchmark(fields, unit, *args):
@@ -1412,6 +1422,7 @@ def coverage_extractor(fields, unit, *args):
         df.TestPartition.value,
         df.GoBenchTimeout.value,
         df.ModuleLang.value,
+        df.DockerImage.value,
     )
 )
 def go_bench(fields, unit, *args):
diff --git a/build/scripts/fetch_from_docker_repo.py b/build/scripts/fetch_from_docker_repo.py
new file mode 100644
index 0000000000..4ed118f0d1
--- /dev/null
+++ b/build/scripts/fetch_from_docker_repo.py
@@ -0,0 +1,34 @@
+import json
+import os
+import argparse
+import shutil
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--link', required=True)
+    parser.add_argument('--tag', required=True)
+    parser.add_argument('--output-image-path', required=True)
+    parser.add_argument('--output-info-path', required=True)
+    parser.add_argument('--preloaded-path', required=True)
+
+    return parser.parse_args()
+
+
+def main(args):
+    # assume image was downloaded by runner
+    assert os.path.exists(args.preloaded_path)
+
+    output_dir = os.path.dirname(args.output_image_path)
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    shutil.copyfile(args.preloaded_path, args.output_image_path)
+    info = {
+        'link': args.link,
+        'tag': args.tag,
+    }
+    with open(args.output_info_path, 'w') as f:
+        f.write(json.dumps(info))
+
+if __name__ == '__main__':
+    args = parse_args()
+    exit(main(args))
diff --git a/build/ymake.core.conf b/build/ymake.core.conf
index b7aa4a825e..388705f63d 100644
--- a/build/ymake.core.conf
+++ b/build/ymake.core.conf
@@ -1546,6 +1546,10 @@ macro DATA(Data...) {
     ADD_CHECK(check.data $Data)
 }
 
+DOCKER_IMAGES_VALUE=
+macro DOCKER_IMAGE(Images...) {
+    SET_APPEND(DOCKER_IMAGES_VALUE $Images)
+}
 # tag:test
 ### @usage: DATA_FILES([path...])
 ###