aboutsummaryrefslogtreecommitdiffstats
path: root/build/scripts
diff options
context:
space:
mode:
authorpg <pg@yandex-team.com>2025-02-18 09:50:57 +0300
committerpg <pg@yandex-team.com>2025-02-18 10:14:31 +0300
commitd506a01f2c1668b2a6720099c110304f5b015765 (patch)
treed65a4fdc55ec2c30a4b77018f9886c8bcfd6ef9e /build/scripts
parentb9824e1dc11c30a60d6416b9d345758bc7901544 (diff)
downloadydb-d506a01f2c1668b2a6720099c110304f5b015765.tar.gz
refactor out py2 protobuf herobora after
commit_hash:8241ea2d9e5c3d12ca4c8902cdac23ed6630d4ce
Diffstat (limited to 'build/scripts')
-rw-r--r--build/scripts/fix_msvc_output.py29
-rw-r--r--build/scripts/fix_py2_protobuf.py127
-rw-r--r--build/scripts/link_dyn_lib.py2
-rw-r--r--build/scripts/link_exe.py2
-rw-r--r--build/scripts/ya.make2
5 files changed, 22 insertions, 140 deletions
diff --git a/build/scripts/fix_msvc_output.py b/build/scripts/fix_msvc_output.py
index 49df9aee3b..4a5c31102b 100644
--- a/build/scripts/fix_msvc_output.py
+++ b/build/scripts/fix_msvc_output.py
@@ -1,5 +1,8 @@
import subprocess
-import os, sys
+import os
+import sys
+import json
+
# Explicitly enable local imports
# Don't forget to add imported scripts to inputs of the calling command!
@@ -7,8 +10,6 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
import process_command_files as pcf
import process_whole_archive_option as pwa
-from fix_py2_protobuf import fix_py2
-
def out2err(cmd):
return subprocess.Popen(cmd, stdout=sys.stderr).wait()
@@ -38,13 +39,27 @@ def out2err_cut_first_line(cmd):
if __name__ == '__main__':
- mode = sys.argv[1]
- args, wa_peers, wa_libs = pwa.get_whole_archive_peers_and_libs(pcf.skip_markers(sys.argv[2:]))
+ args = sys.argv[1:]
+ mode = args[0]
+ plugins = []
+
+ if mode == 'link' and '--start-plugins' in args:
+ ib = args.index('--start-plugins')
+ ie = args.index('--end-plugins')
+ plugins = args[ib + 1:ie]
+ args = args[:ib] + args[ie + 1:]
+
+ for p in plugins:
+ res = subprocess.check_output([sys.executable, p] + args).decode().strip()
+
+ if res:
+ args = json.loads(res)
+
+ args, wa_peers, wa_libs = pwa.get_whole_archive_peers_and_libs(pcf.skip_markers(args[1:]))
cmd = pwa.ProcessWholeArchiveOption('WINDOWS', wa_peers, wa_libs).construct_cmd(args)
run = out2err
if mode in ('cl', 'ml'):
# First line of cl.exe and ml64.exe stdout is useless: it prints input file
run = out2err_cut_first_line
- if mode == 'link':
- cmd = fix_py2(cmd, have_comand_files=True, prefix='', suffix='lib')
+
sys.exit(run(cmd))
diff --git a/build/scripts/fix_py2_protobuf.py b/build/scripts/fix_py2_protobuf.py
deleted file mode 100644
index 1f5f86c26f..0000000000
--- a/build/scripts/fix_py2_protobuf.py
+++ /dev/null
@@ -1,127 +0,0 @@
-import subprocess
-import os, sys
-
-# Explicitly enable local imports
-# Don't forget to add imported scripts to inputs of the calling command!
-sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-import process_command_files as pcf
-
-
-def run(*args):
- # print >>sys.stderr, args
- return subprocess.check_output(list(args), shell=False).strip()
-
-
-def gen_renames_1(d):
- for l in d.split('\n'):
- l = l.strip()
-
- if ' ' in l:
- yield l.split(' ')[-1]
-
-
-def gen_renames_2(p, d):
- for s in gen_renames_1(d):
- """
- Since clang-17, the option -fsanitize-address-globals-dead-stripping
- has been enabled by default. Due to this, we have broken optimization
- that merges calls to the `asan.module_ctor` function, as we are renaming
- symbols with a prefix of 'py2_'. When this flag is enabled, and
- the functions are not merged, false-positive ODR (One Definition Rule)
- violations occur on objects such as `typeinfo std::exception`, because
- the runtime is trying to handle global objects that have already been handled.
- """
- if 'asan_globals' in s:
- continue
- yield s + ' ' + p + s
-
-
-def gen_renames(p, d):
- return '\n'.join(gen_renames_2(p, d)).strip() + '\n'
-
-
-def rename_syms(where, ret, libs):
- p = 'py2_'
-
- # join libs
- run(where + 'llvm-ar', 'qcL', ret, *libs)
-
- # find symbols to rename
- syms = run(where + 'llvm-nm', '--extern-only', '--defined-only', '-A', ret)
-
- # prepare rename plan
- renames = gen_renames(p, syms)
-
- with open('syms', 'w') as f:
- f.write(renames)
-
- # rename symbols
- run(where + 'llvm-objcopy', '--redefine-syms=syms', ret)
-
- # back-rename some symbols
- args = [
- where + 'llvm-objcopy',
- '--redefine-sym',
- p + 'init_api_implementation=init6google8protobuf8internal19_api_implementation',
- '--redefine-sym',
- p + 'init_message=init6google8protobuf5pyext8_message',
- '--redefine-sym',
- p + 'init6google8protobuf8internal19_api_implementation=init6google8protobuf8internal19_api_implementation',
- '--redefine-sym',
- p + 'init6google8protobuf5pyext8_message=init6google8protobuf5pyext8_message',
- '--redefine-sym',
- p + '_init6google8protobuf8internal19_api_implementation=_init6google8protobuf8internal19_api_implementation',
- '--redefine-sym',
- p + '_init6google8protobuf5pyext8_message=_init6google8protobuf5pyext8_message',
- ret,
- ]
-
- run(*args)
- return ret
-
-
-def fix_py2(cmd, have_comand_files=False, prefix='lib', suffix='a'):
- args = cmd
- if have_comand_files:
- args = pcf.get_args(cmd)
- if 'protobuf_old' not in str(args):
- return cmd
-
- py2_libs = [prefix + 'contrib-libs-protobuf_old.' + suffix, prefix + 'pypython-protobuf-py2.' + suffix]
-
- def need_rename(x):
- for v in py2_libs:
- if v in x:
- return True
-
- return False
-
- old = []
- lib = []
-
- where = os.path.dirname(cmd[0]) + '/'
-
- for x in args:
- if need_rename(x):
- lib.append(x)
- else:
- old.append(x)
-
- name = rename_syms(where, 'libprotoherobora.' + suffix, lib)
-
- if not have_comand_files:
- return old + [name]
-
- for file in cmd:
- if pcf.is_cmdfile_arg(file):
- cmd_file_path = pcf.cmdfile_path(file)
- args = pcf.read_from_command_file(cmd_file_path)
- if not 'protobuf_old' in str(args):
- continue
- with open(cmd_file_path, 'w') as afile:
- for arg in args:
- if not need_rename(arg):
- afile.write(arg + '\n')
- afile.write(name)
-
- return cmd
diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py
index f8e757a3c0..650f198f3c 100644
--- a/build/scripts/link_dyn_lib.py
+++ b/build/scripts/link_dyn_lib.py
@@ -15,7 +15,6 @@ import thinlto_cache
import link_exe
from process_whole_archive_option import ProcessWholeArchiveOption
-from fix_py2_protobuf import fix_py2
def shlex_join(cmd):
@@ -245,7 +244,6 @@ if __name__ == '__main__':
cmd = fix_blas_resolving(args)
cmd = fix_cmd(opts.arch, cmd)
- cmd = fix_py2(cmd)
if opts.dynamic_cuda:
cmd = fix_cmd_for_dynamic_cuda(cmd)
diff --git a/build/scripts/link_exe.py b/build/scripts/link_exe.py
index cc47d689f3..e48ecb4996 100644
--- a/build/scripts/link_exe.py
+++ b/build/scripts/link_exe.py
@@ -14,7 +14,6 @@ import process_command_files as pcf
import thinlto_cache
from process_whole_archive_option import ProcessWholeArchiveOption
-from fix_py2_protobuf import fix_py2
def get_leaks_suppressions(cmd):
@@ -353,7 +352,6 @@ if __name__ == '__main__':
args = pcf.skip_markers(args)
cmd = fix_blas_resolving(args)
- cmd = fix_py2(cmd)
cmd = remove_excessive_flags(cmd)
cmd = fix_sanitize_flag(cmd, opts)
diff --git a/build/scripts/ya.make b/build/scripts/ya.make
index 24998211c3..e59871172b 100644
--- a/build/scripts/ya.make
+++ b/build/scripts/ya.make
@@ -17,7 +17,6 @@ IF (PY2)
fetch_from_mds.py
fetch_from_sandbox.py
fetch_resource.py
- fix_py2_protobuf.py
gen_java_codenav_entry.py
gen_py3_reg.py
go_tool.py
@@ -71,7 +70,6 @@ ELSEIF (PY3)
find_time_trace.py
fix_java_command_file_cp.py
fix_msvc_output.py
- fix_py2_protobuf.py
fs_tools.py
gen_aar_gradle_script.py
gen_java_codenav_protobuf.py
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-01 12:49:16 +0000