Add renaming py2-proto-symbols to all needed linking scripts

b7ff75f77e1a1818aac0062ec06948dad92af1bc

8 files changed, 132 insertions, 92 deletions

diff --git a/build/conf/linkers/ld.conf b/build/conf/linkers/ld.conf
index 809d02d387..504d9f715c 100644
--- a/build/conf/linkers/ld.conf
+++ b/build/conf/linkers/ld.conf
@@ -192,7 +192,7 @@ GENERATE_MF_CMD=\
 LINK_SCRIPT_EXE_FLAGS=
 
 REAL_LINK_EXE_CMDLINE =\
-    $YMAKE_PYTHON ${input:"build/scripts/link_exe.py"} \
+    $YMAKE_PYTHON ${input:"build/scripts/link_exe.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"} \
     --source-root $ARCADIA_ROOT
 REAL_LINK_EXE_CMDLINE+=--clang-ver $CLANG_VER
 REAL_LINK_EXE_CMDLINE+=$_LD_LINKER_OUTPUT
@@ -220,7 +220,7 @@ REAL_LINK_EXE_TIDY=\
 # Executable Shared Library
 
 REAL_LINK_EXEC_DYN_LIB_CMDLINE =\
-    $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} \
+    $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"}\
     --target $TARGET
 REAL_LINK_EXEC_DYN_LIB_CMDLINE+=$_LD_LINKER_OUTPUT
 REAL_LINK_EXEC_DYN_LIB_CMDLINE+=\
@@ -244,7 +244,7 @@ REAL_LINK_EXEC_DYN_LIB=$REAL_LINK_EXEC_DYN_LIB_IMPL($_WHOLE_ARCHIVE_PEERS_VALUE)
 LINK_DYN_LIB_FLAGS=
 
 REAL_LINK_DYN_LIB_CMDLINE =\
-    $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} \
+    $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"} \
     --target $TARGET
 REAL_LINK_DYN_LIB_CMDLINE+=$_LD_LINKER_OUTPUT
 REAL_LINK_DYN_LIB_CMDLINE+=\
diff --git a/build/conf/linkers/msvc_linker.conf b/build/conf/linkers/msvc_linker.conf
index d0e1db7098..5f38076857 100644
--- a/build/conf/linkers/msvc_linker.conf
+++ b/build/conf/linkers/msvc_linker.conf
@@ -138,7 +138,10 @@ when ($_UNDER_WINE_LIB == "yes") {
     LIB_WRAPPER=
 }
 
-LINK_WRAPPER_DYNLIB=${YMAKE_PYTHON} ${input:"build/scripts/link_dyn_lib.py"} --arch WINDOWS --target $TARGET
+LINK_WRAPPER_DYNLIB=\
+    ${YMAKE_PYTHON} ${input:"build/scripts/link_dyn_lib.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"} \
+    --arch WINDOWS \
+    --target $TARGET
 
 EXPORTS_VALUE=
 LINK_IMPLIB_VALUE=
diff --git a/build/scripts/fix_msvc_output.py b/build/scripts/fix_msvc_output.py
index 183a442e1f..c9694afbc8 100644
--- a/build/scripts/fix_msvc_output.py
+++ b/build/scripts/fix_msvc_output.py
@@ -4,6 +4,8 @@ import sys
 import process_command_files as pcf
 import process_whole_archive_option as pwa
 
+from fix_py2_protobuf import fix_py2
+
 
 def out2err(cmd):
     return subprocess.Popen(cmd, stdout=sys.stderr).wait()
@@ -40,4 +42,6 @@ if __name__ == '__main__':
     if mode in ('cl', 'ml'):
         # First line of cl.exe and ml64.exe stdout is useless: it prints input file
         run = out2err_cut_first_line
+    if mode == 'link':
+        cmd = fix_py2(cmd, have_comand_files=True, prefix='', suffix='lib')
     sys.exit(run(cmd))
diff --git a/build/scripts/fix_py2_protobuf.py b/build/scripts/fix_py2_protobuf.py
new file mode 100644
index 0000000000..06e81ed648
--- /dev/null
+++ b/build/scripts/fix_py2_protobuf.py
@@ -0,0 +1,113 @@
+import subprocess
+import os
+
+import process_command_files as pcf
+
+
+def run(*args):
+    # print >>sys.stderr, args
+    return subprocess.check_output(list(args), shell=False).strip()
+
+
+def gen_renames_1(d):
+    for l in d.split('\n'):
+        l = l.strip()
+
+        if ' ' in l:
+            yield l.split(' ')[-1]
+
+
+def gen_renames_2(p, d):
+    for s in gen_renames_1(d):
+        yield s + ' ' + p + s
+
+
+def gen_renames(p, d):
+    return '\n'.join(gen_renames_2(p, d)).strip() + '\n'
+
+
+def rename_syms(where, ret, libs):
+    p = 'py2_'
+
+    # join libs
+    run(where + 'llvm-ar', 'qcL', ret, *libs)
+
+    # find symbols to rename
+    syms = run(where + 'llvm-nm', '--extern-only', '--defined-only', '-A', ret)
+
+    # prepare rename plan
+    renames = gen_renames(p, syms)
+
+    with open('syms', 'w') as f:
+        f.write(renames)
+
+    # rename symbols
+    run(where + 'llvm-objcopy', '--redefine-syms=syms', ret)
+
+    # back-rename some symbols
+    args = [
+        where + 'llvm-objcopy',
+        '--redefine-sym',
+        p + 'init_api_implementation=init6google8protobuf8internal19_api_implementation',
+        '--redefine-sym',
+        p + 'init_message=init6google8protobuf5pyext8_message',
+        '--redefine-sym',
+        p + 'init6google8protobuf8internal19_api_implementation=init6google8protobuf8internal19_api_implementation',
+        '--redefine-sym',
+        p + 'init6google8protobuf5pyext8_message=init6google8protobuf5pyext8_message',
+        '--redefine-sym',
+        p + '_init6google8protobuf8internal19_api_implementation=_init6google8protobuf8internal19_api_implementation',
+        '--redefine-sym',
+        p + '_init6google8protobuf5pyext8_message=_init6google8protobuf5pyext8_message',
+        ret,
+    ]
+
+    run(*args)
+    return ret
+
+
+def fix_py2(cmd, have_comand_files=False, prefix='lib', suffix='a'):
+    args = cmd
+    if have_comand_files:
+        args = pcf.get_args(cmd)
+    if 'protobuf_old' not in str(args):
+        return cmd
+
+    py2_libs = [prefix + 'contrib-libs-protobuf_old.' + suffix, prefix + 'pypython-protobuf-py2.' + suffix]
+
+    def need_rename(x):
+        for v in py2_libs:
+            if v in x:
+                return True
+
+        return False
+
+    old = []
+    lib = []
+
+    where = os.path.dirname(cmd[0]) + '/'
+
+    for x in args:
+        if need_rename(x):
+            lib.append(x)
+        else:
+            old.append(x)
+
+    name = rename_syms(where, 'libprotoherobora.' + suffix, lib)
+
+    if not have_comand_files:
+        return old + [name]
+
+    for file in cmd:
+        if pcf.is_cmdfile_arg(file):
+            cmd_file_path = pcf.cmdfile_path(file)
+            args = pcf.read_from_command_file(cmd_file_path)
+            if not 'protobuf_old' in str(args):
+                continue
+            with open(cmd_file_path, 'w') as afile:
+                for arg in args:
+                    if not need_rename(arg):
+                        afile.write(arg + '\n')
+                afile.write(name)
+
+    return cmd
diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py
index 1049089054..6171e83179 100644
--- a/build/scripts/link_dyn_lib.py
+++ b/build/scripts/link_dyn_lib.py
@@ -8,6 +8,8 @@ import pipes
 
 from process_whole_archive_option import ProcessWholeArchiveOption
 
+from fix_py2_protobuf import fix_py2
+
 
 def shlex_join(cmd):
     # equivalent to shlex.join() in python 3
@@ -225,6 +227,7 @@ if __name__ == '__main__':
 
     cmd = fix_blas_resolving(args)
     cmd = fix_cmd(opts.arch, cmd)
+    cmd = fix_py2(cmd)
 
     if opts.musl:
         cmd = fix_cmd_for_musl(cmd)
diff --git a/build/scripts/link_exe.py b/build/scripts/link_exe.py
index fca118101c..259d2b27d7 100644
--- a/build/scripts/link_exe.py
+++ b/build/scripts/link_exe.py
@@ -10,6 +10,8 @@ import process_command_files as pcf
 
 from process_whole_archive_option import ProcessWholeArchiveOption
 
+from fix_py2_protobuf import fix_py2
+
 
 def get_leaks_suppressions(cmd):
     supp, newcmd = [], []
@@ -302,93 +304,6 @@ def parse_args():
     return parser.parse_args()
 
 
-def run(*args):
-    # print >>sys.stderr, args
-    return subprocess.check_output(list(args), shell=False).strip()
-
-
-def gen_renames_1(d):
-    for l in d.split('\n'):
-        l = l.strip()
-
-        if ' ' in l:
-            yield l.split(' ')[-1]
-
-def gen_renames_2(p, d):
-    for s in gen_renames_1(d):
-        yield s + ' ' + p + s
-
-def gen_renames(p, d):
-    return '\n'.join(gen_renames_2(p, d)).strip() + '\n'
-
-def rename_syms(where, ret, libs):
-    p = 'py2_'
-
-    # join libs
-    run(where + 'llvm-ar', 'qLc', ret, *libs)
-
-    # find symbols to rename
-    syms = run(where + 'llvm-nm', '--extern-only', '--defined-only', '-A', ret)
-
-    # prepare rename plan
-    renames = gen_renames(p, syms)
-
-    with open('syms', 'w') as f:
-        f.write(renames)
-
-    # rename symbols
-    run(where + 'llvm-objcopy', '--redefine-syms=syms', ret)
-
-    # back-rename some symbols
-    args = [
-        where + 'llvm-objcopy',
-        '--redefine-sym',
-        p + 'init_api_implementation=init6google8protobuf8internal19_api_implementation',
-        '--redefine-sym',
-        p + 'init_message=init6google8protobuf5pyext8_message',
-        '--redefine-sym',
-        p + 'init6google8protobuf8internal19_api_implementation=init6google8protobuf8internal19_api_implementation',
-        '--redefine-sym',
-        p + 'init6google8protobuf5pyext8_message=init6google8protobuf5pyext8_message',
-        '--redefine-sym',
-        p + '_init6google8protobuf8internal19_api_implementation=_init6google8protobuf8internal19_api_implementation',
-        '--redefine-sym',
-        p + '_init6google8protobuf5pyext8_message=_init6google8protobuf5pyext8_message',
-        ret
-    ]
-
-    run(*args)
-
-    return ret
-
-
-def fix_py2(cmd):
-    if 'protobuf_old' not in str(cmd):
-        return cmd
-
-    def my(x):
-        for v in ['libcontrib-libs-protobuf_old.a', 'libpypython-protobuf-py2.a']:
-            if v in x:
-                return True
-
-        return False
-
-    old = []
-    lib = []
-
-    where = ''
-
-    for x in cmd:
-        if '/clang++' in x:
-            where = os.path.dirname(x) + '/'
-
-        if my(x):
-            lib.append(x)
-        else:
-            old.append(x)
-
-    return old + [rename_syms(where, 'libprotoherobora.a', lib)]
-
 if __name__ == '__main__':
     opts, args = parse_args()
     args = pcf.skip_markers(args)
diff --git a/build/scripts/ya.make b/build/scripts/ya.make
index d7e2411e72..aa51191383 100644
--- a/build/scripts/ya.make
+++ b/build/scripts/ya.make
@@ -18,6 +18,7 @@ IF (PY2)
         fetch_from_npm.py
         fetch_from_sandbox.py
         fetch_resource.py
+        fix_py2_protobuf.py
         gen_java_codenav_entry.py
         gen_py3_reg.py
         go_tool.py
@@ -74,6 +75,7 @@ ELSEIF (PY3)
         find_time_trace.py
         fix_java_command_file_cp.py
         fix_msvc_output.py
+        fix_py2_protobuf.py
         fs_tools.py
         gen_aar_gradle_script.py
         gen_java_codenav_protobuf.py
diff --git a/build/ymake.core.conf b/build/ymake.core.conf
index 119f9c8957..023675cc67 100644
--- a/build/ymake.core.conf
+++ b/build/ymake.core.conf
@@ -523,7 +523,7 @@ XARGS=$YMAKE_PYTHON ${input:"build/scripts/xargs.py"} -- ${BINDIR}/__args
 
 WRITER_PY=$YMAKE_PYTHON ${input:"build/scripts/writer.py"} ${hide;input:"build/scripts/process_command_files.py"}
 FS_TOOLS=$YMAKE_PYTHON3 ${input:"build/scripts/fs_tools.py"} ${hide;input:"build/scripts/process_command_files.py"}
-FIX_MSVC_OUTPUT=${YMAKE_PYTHON} ${input:"build/scripts/fix_msvc_output.py"} ${hide;input:"build/scripts/process_command_files.py"}
+FIX_MSVC_OUTPUT=${YMAKE_PYTHON} ${input:"build/scripts/fix_msvc_output.py"} ${hide;input:"build/scripts/process_command_files.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"}
 _PROCESS_WHOLE_ARCHIVE_SCRIPT=${hide;input:"build/scripts/process_command_files.py"} ${hide;input:"build/scripts/process_whole_archive_option.py"}
 
 COPY_CMD=$FS_TOOLS copy