Mach-o objcopy resource insertion support (preparation only)

commit_hash:c8efa31c9cc0e080a87437782707506a75fbf48c

1 files changed, 76 insertions, 35 deletions

diff --git a/build/scripts/objcopy.py b/build/scripts/objcopy.py
index b25276a124a..d6eccf97b7e 100644
--- a/build/scripts/objcopy.py
+++ b/build/scripts/objcopy.py
@@ -29,6 +29,8 @@ class LLVMResourceInserter:
 
         self.tmpdir = tempfile.mkdtemp()
 
+        self.SECTION_NAME = '__DATA,__res_holder' if 'apple' in args.target else '.rodata.__res_holder'
+
     def __enter__(self):
         return self
 
@@ -37,48 +39,88 @@ class LLVMResourceInserter:
 
     PackedArgs = namedtuple("PackedArgs", ["infile_path", "comressed_file_path", "symbol_name"])
 
-    def _insert_files(self, infos: list[PackedArgs], output_obj: str) -> None:
+    def _compress_files(self, infos: list[PackedArgs]) -> list[int]:
         if len(infos) == 0:
-            return
-
+            return [0]
         # Compress resources
         cmd = [self.compressor, "--compress-only"]
         for inserted_file, compressed_file, _ in infos:
             cmd.extend([inserted_file, compressed_file])
         call(cmd)
 
-        # Put resources into .o file
-        infos = [(zipped_file, os.path.getsize(zipped_file), symbol_name) for (_, zipped_file, symbol_name) in infos]
-        cmd = [self.objcopy]
-        # NOTE: objcopy does not distinguish the order of arguments.
-        for fname, fsize, sym in infos:
-            section_name = f'.rodata.{sym}'
-            cmd.extend(
-                [
-                    f'--add-section={section_name}={fname}',
-                    f'--set-section-flags={section_name}=readonly',
-                    f'--add-symbol={sym}={section_name}:0,global',
-                    f'--add-symbol={sym}_end={section_name}:{fsize},global',
-                ]
-            )
-        cmd.extend([output_obj])
-        call(cmd)
+        # Return sizes of compressed files
+        return [os.path.getsize(zipped_file) for (_, zipped_file, _) in infos]
 
     @staticmethod
-    def flat_merge_cpp(outs, output):
+    def flat_merge_obj(outs, output):
         if len(outs) == 1:
             shutil.move(outs[0], output)
             return
 
-        with open(output, 'w') as fout:
+        with open(output, 'wb') as fout:
             for fname in outs:
-                with open(fname, 'r') as fin:
+                with open(fname, 'rb') as fin:
                     shutil.copyfileobj(fin, fout)
         return
 
+    def _insert_files(self, infos: list[PackedArgs], output_obj: str) -> None:
+        if len(infos) == 0:
+            return
+
+        # Merge files into one
+        compressed_files = [compressed_file for _, compressed_file, _ in infos]
+        merged_output = os.path.join(self.tmpdir, 'merged.zstd')
+        self.flat_merge_obj(compressed_files, merged_output)
+
+        # Update section content
+        cmd = [self.objcopy, f'--update-section={self.SECTION_NAME}={merged_output}', output_obj]
+        call(cmd)
+
+    def _gen_prefix(self, infos: list[PackedArgs], f_sizes: list[int]) -> str:
+        TOTAL_SIZE = sum(f_sizes)
+        if len(infos) == 0 or TOTAL_SIZE == 0:
+            return ''
+
+        syms = []
+        for packed_args in infos:
+            syms += [sym for (_, _, sym) in packed_args]
+
+        cumulative_pos = [0] * (len(f_sizes) + 1)
+        for i, sz in enumerate(f_sizes):
+            cumulative_pos[i + 1] = cumulative_pos[i] + sz
+        cumulative_pos.pop()
+
+        SECTION_ATTR = f'__attribute__((section("{self.SECTION_NAME}")))'
+        HOLDER_VARIABLE = 'resource_holder'
+        generated_return = f'static {SECTION_ATTR} const char {HOLDER_VARIABLE}[{TOTAL_SIZE}] = {{0}};\n'
+        template = '#define {sym}      ({var} + {begin_offset})\n'
+        template += '#define {sym}_end  ({var} + {end_offset})\n'
+        for sym, start_offset, size in zip(syms, cumulative_pos, f_sizes):
+            generated_return += template.format(
+                sym=sym, begin_offset=start_offset, end_offset=start_offset + size, var=HOLDER_VARIABLE
+            )
+
+        return generated_return
+
     def insert_resources(self, kv_files, kv_strings):
         kv_files = list(kv_files)
 
+        # Step 1: Compress files and save its sizes
+        file_sizes = []
+        infos = [[]]
+        estimated_cmd_len = 0
+        LIMIT = 6000
+        for idx, (path, key) in enumerate(kv_files):
+            packed_args = (path, os.path.join(self.tmpdir, f'{idx}.zstd'), self.mangler(key))
+            infos[-1].append(packed_args)
+            estimated_cmd_len += len(path)
+            if estimated_cmd_len > LIMIT:
+                infos.append([])
+                estimated_cmd_len = 0
+        for packed_args in infos:
+            file_sizes += self._compress_files(packed_args)
+
+        # Introduce custom tmp_filename generator for keeping object-file hash the same
         def tmp_file_generator():
             idx = 0
             base = os.path.basename(self.obj_out)
@@ -86,7 +128,7 @@ class LLVMResourceInserter:
                 yield f'{base}_{str(idx)}'
                 idx += 1
 
-        # Generate resource registration cpp code & compile it
+        # Step 2: Generate resource registration cpp code & compile it
         with patch.object(tempfile, "_get_candidate_names", tmp_file_generator), tempfile.NamedTemporaryFile(
             suffix='.cc'
         ) as dummy_src:
@@ -97,21 +139,20 @@ class LLVMResourceInserter:
                 cmd.extend([path, key])
             call(cmd)
 
+            # Generate code for real char arrays, that will hold the resource
+            generated_source_text = ''
+            with open(dummy_src.name, 'rt') as f:
+                generated_source_text = f.read()
+            generated_source_text = generated_source_text.replace('extern "C" const char', '// extern "C" const char')
+            generated_source_text = self._gen_prefix(infos, file_sizes) + generated_source_text
+            with open(dummy_src.name, 'wt') as f:
+                f.write(generated_source_text)
+
             # Compile
             call([self.cxx, dummy_src.name, *self.target_flags, '-c', '-o', self.obj_out])
 
-        # Put files
-        infos = [[]]
-        estimated_cmd_len = 0
-        LIMIT = 6000
-        for idx, (path, key) in enumerate(kv_files):
-            packed_args = (path, os.path.join(self.tmpdir, f'{idx}.zstd'), self.mangler(key))
-            infos[-1].append(packed_args)
-            estimated_cmd_len += len(path)
-            if estimated_cmd_len > LIMIT:
-                infos.append([])
-                estimated_cmd_len = 0
-        for packed_args in infos:
+        # Step 3: Put files into object
+        if sum(file_sizes) > 0:
             self._insert_files(packed_args, self.obj_out)
 
         return self.obj_out