Generate cmake

Generate cmake

Pull Request resolved: #272

3 files changed, 360 insertions, 0 deletions

diff --git a/.github/check_dirs.sh b/.github/check_dirs.sh
index ba1e685a37..edd4bc9d53 100755
--- a/.github/check_dirs.sh
+++ b/.github/check_dirs.sh
@@ -14,6 +14,7 @@ declare -A top_dirs=(
     [.github/]=1,
     [library/]=1,
     [tools/]=1,
+    [scripts/]=1,
 )
 
 cd $GIT_URL
diff --git a/generate_cmake b/generate_cmake
new file mode 100755
index 0000000000..78a8d1f7ea
--- /dev/null
+++ b/generate_cmake
@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import shutil
+import subprocess
+import multiprocessing
+import json
+import stat
+import filecmp
+import urllib.request
+from argparse import ArgumentParser
+
+
+def mkdir(path):
+    try:
+        os.mkdir(path)
+    except FileExistsError as e:
+        pass
+
+
+def remove_file(path):
+    try:
+        os.remove(path)
+    except FileNotFoundError as e:
+        pass
+
+
+def compare_files(lhs_file_path, rhs_file_path):
+    try:
+        return filecmp.cmp(lhs_file_path, rhs_file_path)
+    except FileNotFoundError as e:
+        return False
+
+
+def rmtree(path):
+    try:
+        shutil.rmtree(path)
+    except FileNotFoundError as e:
+        pass
+
+
+def make_file_executable(file_path):
+    st = os.stat(file_path)
+    os.chmod(file_path, st.st_mode + stat.S_IEXEC)
+
+
+def get_binary_id(resource_file_path):
+    with open(resource_file_path) as file:
+        ymake_file_json = json.load(file)
+
+    linux_uri = str(ymake_file_json["by_platform"]["linux"]["uri"])
+    return linux_uri.split(":")[1]
+
+
+def download_binary(root_path, binary_name, binary_path):
+    root_binary_resource_file_path = (
+        f"{root_path}/build/external_resources/{binary_name}/resources.json"
+    )
+    tmp_binary_resource_file_path = binary_path + "_resources.json"
+
+    if compare_files(root_binary_resource_file_path, tmp_binary_resource_file_path):
+        print(f"Use {binary_name} binary from cache {binary_path}")
+    else:
+        binary_id = get_binary_id(root_binary_resource_file_path)
+        devtools_registry_s3_url = "https://devtools-registry.s3.yandex.net"
+        devtools_registry_s3_binary_url = f"{devtools_registry_s3_url}/{binary_id}"
+        print(
+            f"Download {binary_name} binary from {devtools_registry_s3_binary_url} into {binary_path}"
+        )
+        remove_file(binary_path)
+        urllib.request.urlretrieve(devtools_registry_s3_binary_url, binary_path)
+        make_file_executable(binary_path)
+
+    shutil.copy(root_binary_resource_file_path, tmp_binary_resource_file_path)
+
+
+def generate_graph_for_platform(generate_graph_for_platform):
+    platform = generate_graph_for_platform[0]
+    generate_graph_command = generate_graph_for_platform[1]
+
+    output = subprocess.check_output(
+        generate_graph_command, stderr=subprocess.STDOUT, shell=True
+    ).decode("utf-8")
+
+    allowed_error_patterns = [
+        "to directory without ya.make: [[imp]]$S/build/platform/",
+        "to directory without ya.make: [[imp]]$S/build/external_resources/",
+        "could not resolve include file: [[imp]]openssl",
+        "could not resolve include file: [[imp]]zlib",
+        "could not resolve include file: [[imp]]ares.h",
+        "in $B/contrib/libs/openssl/",
+        "in $B/contrib/libs/zlib",
+        "in $B/contrib/libs/c-ares",
+        "in $B/contrib/libs/libc_compat/ubuntu_14/liblibs-libc_compat-ubuntu_14.a",
+        "in $B/contrib/libs/linux-headers/libcontrib-libs-linux-headers.a",
+        "in $B/contrib/libs/farmhash/",
+        "in $B/contrib/libs/curl/",
+        "in $B/contrib/libs/libxml/",
+        "in $B/contrib/libs/apache/arrow/",
+        "in $B/contrib/libs/grpc/",
+        "in $S/contrib/tools/protoc/plugins/cpp_styleguide/ya.make",
+        "in $S/contrib/tools/protoc/plugins/grpc_cpp",
+        "in $B/contrib/restricted/boost/",
+        "in $B/library/cpp/charset/",
+        "in $B/library/cpp/uri/",
+        "in $B/library/cpp/unicode/punycode/",
+        "in $B/library/cpp/config/",
+        "in $S/tools/rescompiler/bin/",
+        # Fix
+        "in $B/library/cpp/actors/dnsresolver/ut/library-cpp-actors-dnsresolver-ut",
+        "in $B/ydb/library/pdisk_io/libydb-library-pdisk_io",
+    ]
+
+    if platform == "windows-x86_64":
+        # Fix
+        allowed_error_patterns.append("in $B/ydb/core/tx/tiering/core-tx-tiering")
+        allowed_error_patterns.append(
+            "in $B/ydb/library/yql/providers/s3/serializations/providers-s3-serializations"
+        )
+
+    result_errors = []
+    for line in output.split("\n"):
+        if not line.startswith("Error"):
+            continue
+
+        error_is_allowed = False
+        for allowed_error_pattern in allowed_error_patterns:
+            if allowed_error_pattern in line:
+                error_is_allowed = True
+                break
+
+        if error_is_allowed:
+            continue
+
+        result_errors.append(line)
+
+    return result_errors
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description="Generate CMake files from Ya make files")
+    parser.add_argument("--ymake_bin", help="Path to ymake binary")
+    parser.add_argument("--yexport_bin", help="Path to yexport binary")
+    parser.add_argument("--tmp", help="Path to tmp dir")
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Run script in debug mode"
+    )
+
+    try:
+        args = parser.parse_args()
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+    tmp_folder_path = args.tmp
+    if tmp_folder_path is None:
+        tmp_folder_path = "/tmp/ydb-generate-cmake"
+
+    ymake_binary_path = args.ymake_bin
+    yexport_binary_path = args.yexport_bin
+    debug = args.debug
+
+    ydb_tmp_folder_path = tmp_folder_path + "/ydb"
+    ydb_metadata_folder_path = tmp_folder_path + "/metadata"
+    ydb_bin_folder_path = tmp_folder_path + "/bin"
+    plugins_folder_path = ydb_tmp_folder_path + "/build/plugins"
+
+    mkdir(tmp_folder_path)
+    mkdir(ydb_metadata_folder_path)
+    mkdir(ydb_bin_folder_path)
+
+    root_folder = os.getcwd()
+
+    if ymake_binary_path is None:
+        ymake_binary_path = ydb_bin_folder_path + "/ymake"
+        download_binary(root_folder, "ymake", ymake_binary_path)
+
+    if yexport_binary_path is None:
+        yexport_binary_path = ydb_bin_folder_path + "/yexport"
+        download_binary(root_folder, "yexport", yexport_binary_path)
+
+    rmtree(ydb_tmp_folder_path)
+    shutil.copytree(root_folder, ydb_tmp_folder_path)
+
+    platforms = [
+        ("linux-x86_64", "default-linux-x86_64"),
+        ("linux-aarch64", "default-linux-aarch64"),
+        ("darwin-x86_64", "default-darwin-x86_64"),
+        ("windows-x86_64", "default-win-x86_64"),
+    ]
+
+    generate_graph_for_platform_commands = []
+
+    for platform, target_platform in platforms:
+        print(f"Platform {platform} target platform {target_platform}")
+
+        dump_export_path = f"{ydb_metadata_folder_path}/{platform}.conf"
+        graph_export_path = f"{ydb_metadata_folder_path}/sem.{platform}.json"
+
+        generate_dump_command = f"{root_folder}/scripts/generate_dump.sh {platform} {target_platform} > {dump_export_path}"
+        print(f"Generate dump command {generate_dump_command}")
+
+        subprocess.check_output(generate_dump_command, shell=True)
+
+        # In original script there are targets kikimr/docs/ru/docs_oss ydb ydb/tests/oss/launch library/cpp/actors tools/rescompiler/bin
+        generate_graph_command = f'{ymake_binary_path} --build-root "{ydb_tmp_folder_path}" --config "{dump_export_path}"\
+            --plugins-root "{plugins_folder_path}" --xs --xx --sem-graph --keep-going\
+            ydb ydb/tests/oss/launch library/cpp/actors tools/rescompiler/bin > {graph_export_path}'
+        print(f"Generate graph command {generate_graph_command}")
+
+        generate_graph_for_platform_commands.append((platform, generate_graph_command))
+
+    errors_for_platform = []
+    with multiprocessing.Pool(len(generate_graph_for_platform_commands)) as pool:
+        errors_for_platform = pool.map(
+            generate_graph_for_platform, generate_graph_for_platform_commands
+        )
+
+    for index, (platform, target_platform) in enumerate(platforms):
+        errors_for_platform_size = len(errors_for_platform[index])
+        if errors_for_platform_size == 0:
+            continue
+
+        print(
+            f"Found {errors_for_platform_size} errors for platform {platform}",
+            file=sys.stderr,
+        )
+        for error in errors_for_platform[index]:
+            print(error, file=sys.stderr)
+
+        sys.exit(1)
+
+    yexport_command = f"{yexport_binary_path} --export-root \"{ydb_tmp_folder_path}\" --target YDB \
+            --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-x86_64.json'}\" --platforms linux-x86_64 \
+            --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-aarch64.json'}\" --platforms linux-aarch64 \
+            --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-x86_64.json'}\" --platforms darwin-x86_64 \
+            --semantic-graph \"{ydb_metadata_folder_path + '/sem.windows-x86_64.json'}\" --platforms windows-x86_64"
+    print(f"yexport command {yexport_command}")
+
+    yexport_output = subprocess.check_output(
+        yexport_command, stderr=subprocess.STDOUT, shell=True
+    ).decode("utf-8")
+
+    if debug:
+        print("yexport output")
+        print(yexport_output)
+
+    rsync_command = f'rsync --recursive --delete  --perms\
+        --exclude .git --exclude contrib --exclude library/cpp/actors\
+        "{ydb_tmp_folder_path}/" "{root_folder}/"'
+
+    print(f"rsync command {rsync_command}")
+    subprocess.check_output(rsync_command, shell=True)
+
+    sys.exit(0)
diff --git a/scripts/generate_dump.sh b/scripts/generate_dump.sh
new file mode 100755
index 0000000000..28e4067981
--- /dev/null
+++ b/scripts/generate_dump.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+TMP_FOLDER='/tmp/ydb-build-dump'
+
+rm -rf $TMP_FOLDER
+mkdir $TMP_FOLDER
+
+PLATFORM=$1
+TARGET_PLATFORM=$2
+
+PARAMS_FILE=$TMP_FOLDER/params.json
+UNIX_TEMPLATE=$TMP_FOLDER/unix_template.txt
+WIN_TEMPLATE=$TMP_FOLDER/win_template.txt
+
+cat <<'EOF'>$UNIX_TEMPLATE
+{
+  "env": {
+    "CPATH": [""],
+    "LIBRARY_PATH": [""],
+    "SDKROOT": [""]
+  },
+  "params": {
+    "c_compiler": "$$($platform)/bin/clang",
+    "cxx_compiler": "$$($platform)/bin/clang++",
+    "gcc_version": "14.0",
+    "llvm-symbolizer": "$$($platform)/bin/llvm-symbolizer",
+    "match_root": "$platform",
+    "objcopy": "$$($platform)/bin/llvm-objcopy",
+    "strip": "$$($platform)/bin/llvm-strip",
+    "type": "clang",
+    "werror_mode": "all"
+  },
+  "platform": {
+    "host": {
+      "arch": "$arch",
+      "os": "$os",
+      "toolchain": "default",
+      "visible_name": "clang14"
+    },
+    "target": {
+      "arch": "$arch",
+      "os": "$os",
+      "toolchain": "default",
+      "visible_name": "clang14"
+    }
+  }
+}
+EOF
+
+cat <<'EOF'>$WIN_TEMPLATE
+{
+  "params": {
+    "c_compiler": "$$($platform)/bin/Hostx64/x64/cl.exe",
+    "cxx_compiler": "$$($platform)/bin/Hostx64/x64/cl.exe",
+    "cxx_std": "c++latest",
+    "for_ide": "msvs2019",
+    "match_root": "$platform",
+    "type": "msvc",
+    "version": "2019",
+    "werror_mode": "compiler_specific"
+  },
+  "platform": {
+    "host": {
+      "arch": "$arch",
+      "os": "$os",
+      "toolchain": "default",
+      "visible_name": "msvc2019"
+    },
+    "target": {
+      "arch": "$arch",
+      "os": "$os",
+      "toolchain": "default",
+      "visible_name": "msvc2019"
+    }
+  }
+}
+EOF
+
+if [[ $PLATFORM = win* ]]
+then
+    TEMPLATE=$WIN_TEMPLATE
+else
+    TEMPLATE=$UNIX_TEMPLATE
+fi
+
+DUMP_EXPORT_PATH="$TMP_FOLDER/ymake.$PLATFORM.conf"
+
+# generate params for ymake_conf.py
+python3 -c "import sys, string as s; v=sys.argv; p = v[1].replace('-', '_'); o, a = v[2].split('-'); print(s.Template(open('$TEMPLATE').read()).substitute(platform=p.upper(), arch=a, os=o.upper()))" $TARGET_PLATFORM $PLATFORM >$DUMP_EXPORT_PATH.params
+PARAMS=`base64 -w0 $DUMP_EXPORT_PATH.params`
+
+ARCADIA=`realpath .`
+python3 $ARCADIA/build/ymake_conf.py $ARCADIA release no --toolchain-params $PARAMS \
+    -D NO_SVN_DEPENDS=yes -D REPORT_CONFIGURE_PROGRESS=yes -D EXPORT_CMAKE=yes -D TRAVERSE_RECURSE=yes -D TRAVERSE_RECURSE_FOR_TESTS=yes \
+    -D BUILD_LANGUAGES=CPP -D EXPORTED_BUILD_SYSTEM_SOURCE_ROOT='${CMAKE_SOURCE_DIR}' -D EXPORTED_BUILD_SYSTEM_BUILD_ROOT='${CMAKE_BINARY_DIR}' \
+    -D OPENSOURCE=yes -D OPENSOURCE_PROJECT=ydb -D HAVE_CUDA=no -D CUDA_VERSION=0.0 -D USE_PREBUILT_TOOLS=no >$DUMP_EXPORT_PATH
+# append new line
+echo >>$DUMP_EXPORT_PATH
+
+cat $DUMP_EXPORT_PATH
+# cp $DUMP_EXPORT_PATH .
+
+rm -rf $TMP_FOLDER