collect .proto files by PROTO_DESCRIPTIONS

0727af35f2be899dfb58c5e49ed9cd1701be9066

5 files changed, 118 insertions, 8 deletions

diff --git a/build/conf/proto.conf b/build/conf/proto.conf
index d68ec651d3..30263d68a0 100644
--- a/build/conf/proto.conf
+++ b/build/conf/proto.conf
@@ -639,12 +639,12 @@ macro LIST_PROTO(TO="files.proto", Files...) {
 }
 
 # tag:proto
-macro _PROTO_DESC_CMD(File) {
-    .CMD=${cwd;rootdir;input:File} $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_INCLUDE_PATH --descriptor_set_out=${output;suf=.desc:File} --include_source_info $_PROTOC_FLAGS ${input;rootrel:File} ${hide:PROTO_FAKEID}
+macro _PROTO_DESC_RAWPROTO_CMD(File) {
+    .CMD=${cwd;rootdir;input:File} $YMAKE_PYTHON3 ${input:"build/scripts/desc_rawproto_wrapper.py"} --desc-output ${output;suf=.desc:File} --rawproto-output ${output;norel;suf=.rawproto:File} --proto-file ${input;rootrel:File} -- $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_INCLUDE_PATH --include_source_info $_PROTOC_FLAGS ${hide:PROTO_FAKEID}
 }
 
-_PROTO_DESC_MERGE_CMD=$YMAKE_PYTHON ${input:"build/scripts/merge_files.py"} $TARGET $AUTO_INPUT ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"}
-_PROTO_DESC_MERGE_PEERS_CMD=$YMAKE_PYTHON ${input:"build/scripts/merge_files.py"} $TARGET $PEERS ${ext=.protodesc:SRCS_GLOBAL}  ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"}
+_PROTO_DESC_MERGE_CMD=$YMAKE_PYTHON3 ${input:"build/scripts/merge_files.py"} $TARGET ${ext=.desc:AUTO_INPUT} ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"} && ${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON3 ${input:"build/scripts/collect_rawproto.py"} --output ${output;suf=.protosrc:REALPRJNAME} ${rootrel;ext=.rawproto:AUTO_INPUT}
+_PROTO_DESC_MERGE_PEERS_CMD=$YMAKE_PYTHON3 ${input:"build/scripts/merge_files.py"} $TARGET $PEERS ${ext=.protodesc:SRCS_GLOBAL}  ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"} && ${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON3 ${input:"build/scripts/merge_protosrc.py"} --output ${output;suf=.tar:REALPRJNAME} ${rootrel:PEERS}
 
 NEED_GOOGLE_PROTO_PEERDIRS=yes
 
@@ -807,7 +807,7 @@ multimodule PROTO_LIBRARY {
     module DESC_PROTO: _BARE_UNIT {
         .CMD=_PROTO_DESC_MERGE_CMD
         .SEM=IGNORED
-        .EXTS=.desc
+        .EXTS=.desc .rawproto
         .NODE_TYPE=Library
         .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO RESOURCE GO_PROTO_PLUGIN GRPC
 
@@ -869,6 +869,6 @@ _CPP_PROTO_EVLOG_CMDLINE=$_CPP_PROTO_EVLOG_CMD($SRC)
 _CPP_VANILLA_PROTO_CMDLINE=$_CPP_VANILLA_PROTO_CMD($SRC)
 _JAVA_EVLOG_CMDLINE=$_JAVA_EVLOG_CMD($SRC)
 _JAVA_PROTO_CMDLINE=$_JAVA_PROTO_CMD($SRC)
-_PROTO_DESC_CMDLINE=$_PROTO_DESC_CMD($SRC)
+_PROTO_DESC_CMDLINE=$_PROTO_DESC_RAWPROTO_CMD($SRC)
 _PY_EVLOG_CMDLINE=$_PY_EVLOG_CMD($SRC)
 _PY_PROTO_CMDLINE=$_PY_PROTO_CMD($SRC)
diff --git a/build/scripts/collect_rawproto.py b/build/scripts/collect_rawproto.py
new file mode 100644
index 0000000000..d45347f5cc
--- /dev/null
+++ b/build/scripts/collect_rawproto.py
@@ -0,0 +1,37 @@
+import argparse
+import os
+import tarfile
+import stat
+import sys
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--output', required=True)
+    parser.add_argument('args', nargs='*')
+    return parser.parse_args()
+
+
+def main(args):
+    rawprotos = args.args
+    with tarfile.open(args.output, 'w') as fout:
+        for rawproto in sorted(os.path.normpath(r).replace('\\', '/') for r in rawprotos):
+            assert rawproto.endswith('.rawproto')
+            arcname = rawproto[:-len('.rawproto')]
+            with open(rawproto, 'rb') as fin:
+                tarinfo = fout.gettarinfo(rawproto, arcname)
+                tarinfo.mode = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH if tarinfo.mode | stat.S_IXUSR else 0
+                tarinfo.mode = (
+                    tarinfo.mode | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH
+                )
+                tarinfo.mtime = 0
+                tarinfo.uid = 0
+                tarinfo.gid = 0
+                tarinfo.uname = 'dummy'
+                tarinfo.gname = 'dummy'
+                fout.addfile(tarinfo, fin)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(parse_args()))
diff --git a/build/scripts/desc_rawproto_wrapper.py b/build/scripts/desc_rawproto_wrapper.py
new file mode 100644
index 0000000000..94dd90fd3f
--- /dev/null
+++ b/build/scripts/desc_rawproto_wrapper.py
@@ -0,0 +1,34 @@
+import argparse
+import shutil
+import subprocess
+import sys
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--desc-output', required=True)
+    parser.add_argument('--rawproto-output', required=True)
+    parser.add_argument('--proto-file', required=True)
+    parser.add_argument('args', nargs='+')
+
+    return parser.parse_args()
+
+
+def main(args):
+    cmd = list(args.args)
+    cmd.append(f'--descriptor_set_out={args.desc_output}')
+    cmd.append(args.proto_file)
+
+    try:
+        subprocess.run(cmd, stdin=None, stderr=subprocess.STDOUT, text=True, check=True)
+    except subrocess.CalledProcessError as e:
+        sys.stderr.write(f'{e.cmd} returned non-zero exit code {e.returncode}.\n{e.output}\n')
+        return e.returncode
+
+    shutil.copyfile(args.proto_file, args.rawproto_output)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(parse_args()))
diff --git a/build/scripts/merge_files.py b/build/scripts/merge_files.py
index d42d6a2139..7d52a56f0f 100644
--- a/build/scripts/merge_files.py
+++ b/build/scripts/merge_files.py
@@ -2,7 +2,7 @@ import sys
 
 
 if __name__ == "__main__":
-    with open(sys.argv[1], "w") as f:
+    with open(sys.argv[1], "wb") as f:
         for appended in sys.argv[2:]:
-            with open(appended) as a:
+            with open(appended, "rb") as a:
                 f.write(a.read())
diff --git a/build/scripts/merge_protosrc.py b/build/scripts/merge_protosrc.py
new file mode 100644
index 0000000000..55e44c9063
--- /dev/null
+++ b/build/scripts/merge_protosrc.py
@@ -0,0 +1,39 @@
+import argparse
+import os
+import tarfile
+import sys
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--output', required=True)
+    parser.add_argument('args', nargs='+')
+    return parser.parse_args()
+
+
+def main(args):
+    peers = args.args
+
+    compression_mode = ''
+    if args.output.endswith(('.tar.gz', '.tgz')):
+        compression_mode = 'gz'
+    elif args.output.endswith('.bzip2'):
+        compression_mode = 'bz2'
+
+    files = set()
+    with tarfile.open(args.output, f'w:{compression_mode}') as dest:
+        for psrc in [p[:-len('.self.protodesc')]+'.protosrc' for p in peers if p.endswith('.self.protodesc')]:
+            with tarfile.open(psrc, 'r') as src:
+                for tarinfo in [m for m in src.getmembers() if m.name != '']:
+                    if tarinfo.name in files:
+                        continue
+                    files.add(tarinfo.name)
+                    if tarinfo.isdir():
+                        dest.addfile(tarinfo)
+                    else:
+                        dest.addfile(tarinfo, src.extractfile(tarinfo))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(parse_args()))