,Support thinlto caches

ac03c8c8d7beb99b5a8fcda8c0c6bf948ca2bb2e

5 files changed, 77 insertions, 14 deletions

diff --git a/build/conf/linkers/ld.conf b/build/conf/linkers/ld.conf
index 956d409b7d..20dc87f553 100644
--- a/build/conf/linkers/ld.conf
+++ b/build/conf/linkers/ld.conf
@@ -175,6 +175,7 @@ when ($OS_LINUX == "yes") {
 }
 
 _LD_ENV_STYLE=${cwd:ARCADIA_BUILD_ROOT} $TOOLCHAIN_ENV ${hide;kv:"p LD"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"}
+_ROOT_FLAGS=--source-root $ARCADIA_ROOT --build-root $(BUILD_ROOT)
 
 # Program
 GENERATE_MF_CMD=\
@@ -189,30 +190,35 @@ GENERATE_MF_CMD=\
     -Ya,credits ${input:CREDITS_TEXTS_FILE} $CREDITS_FLAGS \
     --ya-end-command-file
 
+_LINK_SCRIPT_IMPORTS=\
+    ${hide;input:"build/scripts/fix_py2_protobuf.py"} \
+    ${hide;input:"build/scripts/thinlto_cache.py"} \
+    $_PROCESS_WHOLE_ARCHIVE_SCRIPT
+
 LINK_SCRIPT_EXE_FLAGS=
 
 REAL_LINK_EXE_CMDLINE =\
-    $YMAKE_PYTHON ${input:"build/scripts/link_exe.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"} \
-    --source-root $ARCADIA_ROOT
+    $YMAKE_PYTHON ${input:"build/scripts/link_exe.py"} \
 REAL_LINK_EXE_CMDLINE+=--clang-ver $CLANG_VER
 REAL_LINK_EXE_CMDLINE+=$_LD_LINKER_OUTPUT
 REAL_LINK_EXE_CMDLINE+=\
+    $_ROOT_FLAGS \
     ${pre=--whole-archive-peers :WHOLE_ARCHIVE_PEERS} \
     ${pre=--whole-archive-libs :_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL} \
     $_LD_ARCH_FLAG \
     $LINK_SCRIPT_EXE_FLAGS \
+    $_THINLTO_CACHE_FLAGS \
     $CXX_COMPILER_OLD \
     $_LD_SRCS_GLOBALS \
     $VCS_C_OBJ $_EXTRA_OBJS $AUTO_INPUT -o $TARGET \
     $_EXE_FLAGS \
-    $_PROCESS_WHOLE_ARCHIVE_SCRIPT \
+    $_LINK_SCRIPT_IMPORTS \
     $_LD_ENV_STYLE
 
 REAL_LINK_EXE_TIDY=\
     $YMAKE_PYTHON ${input:"build/scripts/clang_tidy_arch.py"} \
-    --build-root $ARCADIA_BUILD_ROOT \
-    --source-root $ARCADIA_ROOT \
     --output-file $TARGET \
+    $_ROOT_FLAGS \
     $AUTO_INPUT \
     ${ext=.tidyjson:SRCS_GLOBAL} \
     $_LD_ENV_STYLE
@@ -220,21 +226,23 @@ REAL_LINK_EXE_TIDY=\
 # Executable Shared Library
 
 REAL_LINK_EXEC_DYN_LIB_CMDLINE =\
-    $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"}\
+    $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} \
     --target $TARGET
 REAL_LINK_EXEC_DYN_LIB_CMDLINE+=$_LD_LINKER_OUTPUT
 REAL_LINK_EXEC_DYN_LIB_CMDLINE+=\
+    $_ROOT_FLAGS \
     ${pre=--whole-archive-peers :WHOLE_ARCHIVE_PEERS} \
     ${pre=--whole-archive-libs :_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL} \
     $_LD_ARCH_FLAG \
     $LINK_DYN_LIB_FLAGS \
+    $_THINLTO_CACHE_FLAGS \
     $CXX_COMPILER_OLD \
     $_LD_SRCS_GLOBALS \
     $VCS_C_OBJ $_EXTRA_OBJS $AUTO_INPUT -o $TARGET \
     $_EXEC_SHARED_FLAG \
     $_SONAME_FLAG \
     $_EXE_FLAGS \
-    $_PROCESS_WHOLE_ARCHIVE_SCRIPT \
+    $_LINK_SCRIPT_IMPORTS \
     $_LD_ENV_STYLE
 
 REAL_LINK_EXEC_DYN_LIB=$REAL_LINK_EXEC_DYN_LIB_IMPL($_WHOLE_ARCHIVE_PEERS_VALUE)
@@ -244,21 +252,23 @@ REAL_LINK_EXEC_DYN_LIB=$REAL_LINK_EXEC_DYN_LIB_IMPL($_WHOLE_ARCHIVE_PEERS_VALUE)
 LINK_DYN_LIB_FLAGS=
 
 REAL_LINK_DYN_LIB_CMDLINE =\
-    $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} ${hide;input:"build/scripts/fix_py2_protobuf.py"} \
+    $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} \
     --target $TARGET
 REAL_LINK_DYN_LIB_CMDLINE+=$_LD_LINKER_OUTPUT
 REAL_LINK_DYN_LIB_CMDLINE+=\
     ${pre=--whole-archive-peers :WHOLE_ARCHIVE_PEERS} \
     ${pre=--whole-archive-libs :_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL} \
+    $_ROOT_FLAGS \
     $_LD_ARCH_FLAG \
     $LINK_DYN_LIB_FLAGS \
+    $_THINLTO_CACHE_FLAGS \
     $CXX_COMPILER_OLD \
     $_LD_SRCS_GLOBALS \
     $VCS_C_OBJ $_EXTRA_OBJS $AUTO_INPUT -o $TARGET \
     $_SHARED_FLAG \
     $_SONAME_FLAG \
     $_EXE_FLAGS \
-    $_PROCESS_WHOLE_ARCHIVE_SCRIPT \
+    $_LINK_SCRIPT_IMPORTS \
     $_LD_ENV_STYLE
 
 REAL_LINK_DYN_LIB=$REAL_LINK_DYN_LIB_IMPL($_WHOLE_ARCHIVE_PEERS_VALUE)
@@ -383,3 +393,21 @@ when ($OS_LINUX || $OS_DARWIN || $OS_ANDROID) {
     LIBRESOLV=-lresolv
 }
 PROFFLAG=-pg
+
+_THINLTO_CACHE_FLAGS=
+macro THINLTO_CACHE(file) {
+    DEFAULT(THINLTO_CACHE use)
+    SET(_THINLTO_FILE $file)
+
+    when ($CLANG && $USE_THINLTO == "yes") {
+        select ($THINLTO_CACHE) {
+            "gen" ? {
+                 _THINLTO_CACHE_FLAGS=--thinlto-cache-write --thinlto-cache=${output;pre=out-:_THINLTO_FILE}
+            }
+            "use" ? {
+                 _THINLTO_CACHE_FLAGS=--thinlto-cache=${input:_THINLTO_FILE}
+            }
+        }
+    }
+}
+
diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py
index 6171e83179..3b7c0d3e51 100644
--- a/build/scripts/link_dyn_lib.py
+++ b/build/scripts/link_dyn_lib.py
@@ -6,8 +6,9 @@ import collections
 import optparse
 import pipes
 
-from process_whole_archive_option import ProcessWholeArchiveOption
+import thinlto_cache
 
+from process_whole_archive_option import ProcessWholeArchiveOption
 from fix_py2_protobuf import fix_py2
 
 
@@ -208,6 +209,8 @@ def parse_args():
     parser.add_option('--arch')
     parser.add_option('--target')
     parser.add_option('--soname')
+    parser.add_option('--source-root')
+    parser.add_option('--build-root')
     parser.add_option('--fix-elf')
     parser.add_option('--linker-output')
     parser.add_option('--musl', action='store_true')
@@ -216,6 +219,7 @@ def parse_args():
     parser.add_option('--whole-archive-libs', action='append')
     parser.add_option('--custom-step')
     parser.add_option('--python')
+    thinlto_cache.add_options(parser)
     return parser.parse_args()
 
 
@@ -235,6 +239,7 @@ if __name__ == '__main__':
         cmd = fix_cmd_for_dynamic_cuda(cmd)
 
     cmd = ProcessWholeArchiveOption(opts.arch, opts.whole_archive_peers, opts.whole_archive_libs).construct_cmd(cmd)
+    thinlto_cache.preprocess(opts, cmd)
 
     if opts.custom_step:
         assert opts.python
@@ -247,6 +252,7 @@ if __name__ == '__main__':
 
     proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=stdout)
     proc.communicate()
+    thinlto_cache.postprocess(opts)
 
     if proc.returncode:
         print >> sys.stderr, 'linker has failed with retcode:', proc.returncode
diff --git a/build/scripts/link_exe.py b/build/scripts/link_exe.py
index c1533c4dfc..2b69486183 100644
--- a/build/scripts/link_exe.py
+++ b/build/scripts/link_exe.py
@@ -7,9 +7,9 @@ import optparse
 import textwrap
 
 import process_command_files as pcf
+import thinlto_cache
 
 from process_whole_archive_option import ProcessWholeArchiveOption
-
 from fix_py2_protobuf import fix_py2
 
 
@@ -305,17 +305,18 @@ def parse_args():
     parser.add_option('--custom-step')
     parser.add_option('--python')
     parser.add_option('--source-root')
+    parser.add_option('--build-root')
     parser.add_option('--clang-ver')
     parser.add_option('--dynamic-cuda', action='store_true')
     parser.add_option('--cuda-architectures',
                       help='List of supported CUDA architectures, separated by ":" (e.g. "sm_52:compute_70:lto_90a"')
     parser.add_option('--nvprune-exe')
     parser.add_option('--objcopy-exe')
-    parser.add_option('--build-root')
     parser.add_option('--arch')
     parser.add_option('--linker-output')
     parser.add_option('--whole-archive-peers', action='append')
     parser.add_option('--whole-archive-libs', action='append')
+    thinlto_cache.add_options(parser)
     return parser.parse_args()
 
 
@@ -362,5 +363,8 @@ if __name__ == '__main__':
     else:
         stdout = sys.stdout
 
+    thinlto_cache.preprocess(opts, cmd)
     rc = subprocess.call(cmd, shell=False, stderr=sys.stderr, stdout=stdout)
+    thinlto_cache.postprocess(opts)
+
     sys.exit(rc)
diff --git a/build/scripts/thinlto_cache.py b/build/scripts/thinlto_cache.py
new file mode 100644
index 0000000000..35ab755262
--- /dev/null
+++ b/build/scripts/thinlto_cache.py
@@ -0,0 +1,27 @@
+import os
+import tarfile
+
+
+CACHE_DIR_NAME='thinlto_cache_dir'
+
+
+def add_options(parser):
+    parser.add_option('--thinlto-cache')
+    parser.add_option('--thinlto-cache-write', action='store_true')
+
+def preprocess(opts, cmd):
+    if opts.thinlto_cache:
+        cache_dir = os.path.join(opts.build_root, CACHE_DIR_NAME)
+        cmd +=['-Wl,--thinlto-cache-dir={}'.format(cache_dir)]
+        if opts.thinlto_cache_write:
+            os.mkdir(cache_dir)
+        else:
+            with tarfile.open(opts.thinlto_cache, 'r') as tar:
+                tar.extractall(opts.build_root)
+
+def postprocess(opts):
+    if opts.thinlto_cache:
+        cache_dir = os.path.join(opts.build_root, CACHE_DIR_NAME)
+        if opts.thinlto_cache_write:
+            with tarfile.open(opts.thinlto_cache, 'w:gz') as tar:
+                tar.add(cache_dir, arcname=os.path.basename(cache_dir))
diff --git a/build/ymake.core.conf b/build/ymake.core.conf
index db0977c75d..03b940f50f 100644
--- a/build/ymake.core.conf
+++ b/build/ymake.core.conf
@@ -1110,8 +1110,6 @@ module _LINK_UNIT: _BASE_UNIT {
         LINK_SCRIPT_EXE_FLAGS+=--objcopy-exe $OBJCOPY_TOOL
     }
 
-    LINK_SCRIPT_EXE_FLAGS+=--build-root $(BUILD_ROOT)
-
     when ($OPENSOURCE == "yes" && $AUTOCHECK == "yes") {
         # FIXME: Replace AUTOCHECK == yes with _not a host platform_ check after YMAKE-218
         MODULE_LICENSES_RESTRICTION_TYPES = ALLOW_ONLY