refactor out py2 protobuf herobora after

commit_hash:8241ea2d9e5c3d12ca4c8902cdac23ed6630d4ce

9 files changed, 106 insertions, 27 deletions

diff --git a/build/conf/linkers/ld.conf b/build/conf/linkers/ld.conf
index 01b20cf103..e066567c1a 100644
--- a/build/conf/linkers/ld.conf
+++ b/build/conf/linkers/ld.conf
@@ -205,7 +205,6 @@ GENERATE_MF_CMD=\
     --ya-end-command-file
 
 _LINK_SCRIPT_IMPORTS=\
-    ${hide;input:"build/scripts/fix_py2_protobuf.py"} \
     ${hide;input:"build/scripts/thinlto_cache.py"} \
     $_PROCESS_WHOLE_ARCHIVE_SCRIPT
 
diff --git a/build/conf/linkers/msvc_linker.conf b/build/conf/linkers/msvc_linker.conf
index 8985ed761b..c89dfe6d1d 100644
--- a/build/conf/linkers/msvc_linker.conf
+++ b/build/conf/linkers/msvc_linker.conf
@@ -113,7 +113,7 @@ when ($_UNDER_WINE_LIB == "yes") {
 }
 
 LINK_WRAPPER_DYNLIB=\
-    ${YMAKE_PYTHON} ${input:"build/scripts/link_dyn_lib.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"} \
+    ${YMAKE_PYTHON} ${input:"build/scripts/link_dyn_lib.py"} \
     --arch WINDOWS \
     --target $TARGET
 
@@ -144,6 +144,7 @@ _GENERATE_EXTRA_OBJS=
 # first group need /WHOLEARCHIVE: prefix which will be added in fix_msvc_output.py or run_msvc_wine.py
 # the tail of link commands will be added in the third command file
 _MSVC_SRCS_GLOBALS=\
+--start-plugins ${ext=.pyplugin:SRCS_GLOBAL} --end-plugins \
 --start-wa --ya-start-command-file ${qe;rootrel;ext=.lib:SRCS_GLOBAL} --ya-end-command-file --end-wa \
 --ya-start-command-file ${qe;rootrel;ext=.obj:SRCS_GLOBAL} --ya-end-command-file
 
diff --git a/build/scripts/fix_msvc_output.py b/build/scripts/fix_msvc_output.py
index 49df9aee3b..4a5c31102b 100644
--- a/build/scripts/fix_msvc_output.py
+++ b/build/scripts/fix_msvc_output.py
@@ -1,5 +1,8 @@
 import subprocess
-import os, sys
+import os
+import sys
+import json
+
 
 # Explicitly enable local imports
 # Don't forget to add imported scripts to inputs of the calling command!
@@ -7,8 +10,6 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 import process_command_files as pcf
 import process_whole_archive_option as pwa
 
-from fix_py2_protobuf import fix_py2
-
 
 def out2err(cmd):
     return subprocess.Popen(cmd, stdout=sys.stderr).wait()
@@ -38,13 +39,27 @@ def out2err_cut_first_line(cmd):
 
 
 if __name__ == '__main__':
-    mode = sys.argv[1]
-    args, wa_peers, wa_libs = pwa.get_whole_archive_peers_and_libs(pcf.skip_markers(sys.argv[2:]))
+    args = sys.argv[1:]
+    mode = args[0]
+    plugins = []
+
+    if mode == 'link' and '--start-plugins' in args:
+        ib = args.index('--start-plugins')
+        ie = args.index('--end-plugins')
+        plugins = args[ib + 1:ie]
+        args = args[:ib] + args[ie + 1:]
+
+    for p in plugins:
+        res = subprocess.check_output([sys.executable, p] + args).decode().strip()
+
+        if res:
+            args = json.loads(res)
+
+    args, wa_peers, wa_libs = pwa.get_whole_archive_peers_and_libs(pcf.skip_markers(args[1:]))
     cmd = pwa.ProcessWholeArchiveOption('WINDOWS', wa_peers, wa_libs).construct_cmd(args)
     run = out2err
     if mode in ('cl', 'ml'):
         # First line of cl.exe and ml64.exe stdout is useless: it prints input file
         run = out2err_cut_first_line
-    if mode == 'link':
-        cmd = fix_py2(cmd, have_comand_files=True, prefix='', suffix='lib')
+
     sys.exit(run(cmd))
diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py
index f8e757a3c0..650f198f3c 100644
--- a/build/scripts/link_dyn_lib.py
+++ b/build/scripts/link_dyn_lib.py
@@ -15,7 +15,6 @@ import thinlto_cache
 import link_exe
 
 from process_whole_archive_option import ProcessWholeArchiveOption
-from fix_py2_protobuf import fix_py2
 
 
 def shlex_join(cmd):
@@ -245,7 +244,6 @@ if __name__ == '__main__':
 
     cmd = fix_blas_resolving(args)
     cmd = fix_cmd(opts.arch, cmd)
-    cmd = fix_py2(cmd)
 
     if opts.dynamic_cuda:
         cmd = fix_cmd_for_dynamic_cuda(cmd)
diff --git a/build/scripts/link_exe.py b/build/scripts/link_exe.py
index cc47d689f3..e48ecb4996 100644
--- a/build/scripts/link_exe.py
+++ b/build/scripts/link_exe.py
@@ -14,7 +14,6 @@ import process_command_files as pcf
 import thinlto_cache
 
 from process_whole_archive_option import ProcessWholeArchiveOption
-from fix_py2_protobuf import fix_py2
 
 
 def get_leaks_suppressions(cmd):
@@ -353,7 +352,6 @@ if __name__ == '__main__':
     args = pcf.skip_markers(args)
 
     cmd = fix_blas_resolving(args)
-    cmd = fix_py2(cmd)
     cmd = remove_excessive_flags(cmd)
     cmd = fix_sanitize_flag(cmd, opts)
 
diff --git a/build/scripts/ya.make b/build/scripts/ya.make
index 24998211c3..e59871172b 100644
--- a/build/scripts/ya.make
+++ b/build/scripts/ya.make
@@ -17,7 +17,6 @@ IF (PY2)
         fetch_from_mds.py
         fetch_from_sandbox.py
         fetch_resource.py
-        fix_py2_protobuf.py
         gen_java_codenav_entry.py
         gen_py3_reg.py
         go_tool.py
@@ -71,7 +70,6 @@ ELSEIF (PY3)
         find_time_trace.py
         fix_java_command_file_cp.py
         fix_msvc_output.py
-        fix_py2_protobuf.py
         fs_tools.py
         gen_aar_gradle_script.py
         gen_java_codenav_protobuf.py
diff --git a/build/ymake.core.conf b/build/ymake.core.conf
index d3d40074a6..0ed4ec54c5 100644
--- a/build/ymake.core.conf
+++ b/build/ymake.core.conf
@@ -533,7 +533,7 @@ XARGS=$YMAKE_PYTHON ${input:"build/scripts/xargs.py"} -- ${BINDIR}/__args
 
 WRITER_PY=$YMAKE_PYTHON ${input:"build/scripts/writer.py"} ${hide;input:"build/scripts/process_command_files.py"}
 FS_TOOLS=$YMAKE_PYTHON3 ${input:"build/scripts/fs_tools.py"} ${hide;input:"build/scripts/process_command_files.py"}
-FIX_MSVC_OUTPUT=${YMAKE_PYTHON} ${input:"build/scripts/fix_msvc_output.py"} ${hide;input:"build/scripts/process_command_files.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"}
+FIX_MSVC_OUTPUT=${YMAKE_PYTHON} ${input:"build/scripts/fix_msvc_output.py"} ${hide;input:"build/scripts/process_command_files.py"}
 _PROCESS_WHOLE_ARCHIVE_SCRIPT=${hide;input:"build/scripts/process_command_files.py"} ${hide;input:"build/scripts/process_whole_archive_option.py"}
 
 COPY_CMD=$FS_TOOLS copy
diff --git a/build/scripts/fix_py2_protobuf.py b/contrib/libs/protobuf_old/protoherobora.py
index 1f5f86c26f..9b85a204bc 100644
--- a/build/scripts/fix_py2_protobuf.py
+++ b/contrib/libs/protobuf_old/protoherobora.py
@@ -1,14 +1,58 @@
+#!/usr/bin/env python3
+
 import subprocess
-import os, sys
+import os
+import sys
+import json
+
+
+# TODO: dedup
+def is_cmdfile_arg(arg):
+    # type: (str) -> bool
+    return arg.startswith('@')
+
+
+def cmdfile_path(arg):
+    # type: (str) -> str
+    return arg[1:]
+
+
+def read_from_command_file(arg):
+    # type: (str) -> list[str]
+    with open(arg) as afile:
+        return afile.read().splitlines()
 
-# Explicitly enable local imports
-# Don't forget to add imported scripts to inputs of the calling command!
-sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-import process_command_files as pcf
+
+def skip_markers(args):
+    # type: (list[str]) -> list[str]
+    res = []
+    for arg in args:
+        if arg == '--ya-start-command-file' or arg == '--ya-end-command-file':
+            continue
+        res.append(arg)
+    return res
+
+
+def iter_args(
+        args,  # type: list[str]
+        ):
+    for arg in args:
+        if not is_cmdfile_arg(arg):
+            if arg == '--ya-start-command-file' or arg == '--ya-end-command-file':
+                continue
+            yield arg
+        else:
+            for cmdfile_arg in read_from_command_file(cmdfile_path(arg)):
+                yield cmdfile_arg
+
+
+def get_args(args):
+    # type: (list[str]) -> list[str]
+    return list(iter_args(args))
+# end TODO
 
 
 def run(*args):
-    # print >>sys.stderr, args
     return subprocess.check_output(list(args), shell=False).strip()
 
 
@@ -80,10 +124,20 @@ def rename_syms(where, ret, libs):
     return ret
 
 
+def find_lld(args):
+    for x in args:
+        if 'lld-link' in x:
+            return x
+
+    raise IndexError()
+
+
 def fix_py2(cmd, have_comand_files=False, prefix='lib', suffix='a'):
     args = cmd
+
     if have_comand_files:
-        args = pcf.get_args(cmd)
+        args = get_args(cmd)
+
     if 'protobuf_old' not in str(args):
         return cmd
 
@@ -99,7 +153,10 @@ def fix_py2(cmd, have_comand_files=False, prefix='lib', suffix='a'):
     old = []
     lib = []
 
-    where = os.path.dirname(cmd[0]) + '/'
+    try:
+        where = os.path.dirname(cmd[cmd.index('--objcopy-exe') + 1]) + '/'
+    except ValueError:
+        where = os.path.dirname(find_lld(cmd)) + '/'
 
     for x in args:
         if need_rename(x):
@@ -113,9 +170,9 @@ def fix_py2(cmd, have_comand_files=False, prefix='lib', suffix='a'):
         return old + [name]
 
     for file in cmd:
-        if pcf.is_cmdfile_arg(file):
-            cmd_file_path = pcf.cmdfile_path(file)
-            args = pcf.read_from_command_file(cmd_file_path)
+        if is_cmdfile_arg(file):
+            cmd_file_path = cmdfile_path(file)
+            args = read_from_command_file(cmd_file_path)
             if not 'protobuf_old' in str(args):
                 continue
             with open(cmd_file_path, 'w') as afile:
@@ -125,3 +182,14 @@ def fix_py2(cmd, have_comand_files=False, prefix='lib', suffix='a'):
                 afile.write(name)
 
     return cmd
+
+
+if __name__ == '__main__':
+    args = sys.argv[1:]
+
+    if 'lld-link' in str(args):
+        cmd = fix_py2(args, have_comand_files=True, prefix='', suffix='lib')
+    else:
+        cmd = fix_py2(args)
+
+    sys.stdout.write(json.dumps(cmd))
diff --git a/contrib/libs/protobuf_old/ya.make b/contrib/libs/protobuf_old/ya.make
index 50a3d8cd79..39f6a558a8 100644
--- a/contrib/libs/protobuf_old/ya.make
+++ b/contrib/libs/protobuf_old/ya.make
@@ -2,6 +2,8 @@
 
 LIBRARY()
 
+LD_PLUGIN(protoherobora.py)
+
 LICENSE(
     BSD-3-Clause AND
     Protobuf-License