5295 add precommit check for ydbd size add diff in pr (#5597)

7 files changed, 252 insertions, 6 deletions

diff --git a/.github/actions/build_and_test_ya/action.yml b/.github/actions/build_and_test_ya/action.yml
index 5749b7178e..6f2d321125 100644
--- a/.github/actions/build_and_test_ya/action.yml
+++ b/.github/actions/build_and_test_ya/action.yml
@@ -132,7 +132,8 @@ runs:
         link_threads: ${{ inputs.link_threads }}
         additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
         test_threads: ${{ inputs.test_threads }}
-        
+
+    
     - name: Notify about failed build
       if: ${{ steps.build.outputs.success != 'true' && inputs.run_tests == 'true' && inputs.run_tests_if_build_fails == 'false' }}
       shell: bash
@@ -152,9 +153,44 @@ runs:
       run: |
         set -x
         export build_preset="${{ inputs.build_preset }}" 
+        export commit_git_sha="$(git rev-parse HEAD)"
+
         python3 -m pip install ydb ydb[yc]
         python3 .github/scripts/send_build_stats.py
 
+    - name: show_build_size_diff
+      shell: bash
+      continue-on-error: true
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        set -x
+        export build_preset="${{ inputs.build_preset }}" 
+        export branch_to_compare="$GITHUB_REF_NAME" 
+        export yellow_treshold=102400
+        export red_treshold=2097152 
+        export commit_git_sha="$(git rev-parse HEAD)"
+
+        python3 -m pip install ydb ydb[yc] humanize
+
+        get_sizes_comment_script=.github/scripts/get_build_diff.py
+        comment_raw=`$get_sizes_comment_script`
+        
+        IFS=';;;'
+        read -ra comment_arr <<< "$comment_raw"
+
+        printf "$comment"
+        if [[ ${comment_raw} != "Error"* ]];then
+          color=${comment_arr[0]}
+          replace=$color";;;"
+          comment=${comment_raw/$replace/""}
+          
+          printf "$comment" | .github/scripts/tests/comment-pr.py --color $color
+
+        else
+          echo "Skipped build size difference, comment_raw = ${comment_raw}"
+        fi
+
     - name: comment-if-cancel
       shell: bash
       if: cancelled() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target')
diff --git a/.github/config/ydb_qa_db.ini b/.github/config/ydb_qa_db.ini
new file mode 100644
index 0000000000..f15bb526f8
--- /dev/null
+++ b/.github/config/ydb_qa_db.ini
@@ -0,0 +1,5 @@
+[QA_DB]
+DATABASE_PATH = /ru-central1/b1ggceeul2pkher8vhb6/etnvsjbk7kh1jc6bbfi8
+DATABASE_ENDPOINT = grpcs://lb.etnvsjbk7kh1jc6bbfi8.ydb.mdb.yandexcloud.net:2135
+[YDBD]
+YDBD_PATH = ydb/apps/ydbd/ydbd
\ No newline at end of file
diff --git a/.github/scripts/get_build_diff.py b/.github/scripts/get_build_diff.py
new file mode 100755
index 0000000000..30ba3f46c4
--- /dev/null
+++ b/.github/scripts/get_build_diff.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+from decimal import Decimal, ROUND_HALF_UP
+import get_current_build_size
+import get_main_build_size
+import humanize
+import os
+
+
+# Форматирование числа
+def format_number(num):
+    return humanize.intcomma(num).replace(",", " ")
+
+
+def bytes_to_human_iec(num):
+    return humanize.naturalsize(num, binary=True)
+
+
+def main():
+
+    yellow_treshold = int(os.environ.get("yellow_treshold"))
+    red_treshold = int(os.environ.get("red_treshold"))
+
+    branch = os.environ.get("branch_to_compare")
+    current_pr_commit_sha = os.environ.get("commit_git_sha")
+
+    current_sizes_result = get_current_build_size.get_build_size()
+    main_sizes_result = get_main_build_size.get_build_size()
+
+    if main_sizes_result and current_sizes_result:
+        main_github_sha = main_sizes_result["github_sha"]
+        main_size_bytes = int(main_sizes_result["size_bytes"])
+        main_size_stripped_bytes = int(main_sizes_result["size_stripped_bytes"])
+
+        current_size_bytes = int(current_sizes_result["size_bytes"])
+        current_size_stripped_bytes = int(current_sizes_result["size_stripped_bytes"])
+
+        bytes_diff = current_size_bytes - main_size_bytes
+        diff_perc = Decimal(bytes_diff * 100 / main_size_bytes).quantize(
+            Decimal(".001"), rounding=ROUND_HALF_UP
+        )
+
+        human_readable_size = bytes_to_human_iec(current_size_bytes)
+        human_readable_size_diff = bytes_to_human_iec(bytes_diff)
+
+        if bytes_diff >= 0:
+            sign = "+"
+            if bytes_diff >= red_treshold:
+                color = "red"
+            elif bytes_diff >= yellow_treshold:
+                color = "yellow"
+            else:
+                color = "green"
+        else:
+            sign = ""
+            color = "green"
+
+        comment = (
+            f"merge: {current_pr_commit_sha} ydbd size {human_readable_size} **{sign}{human_readable_size_diff} {sign}{diff_perc}%%** vs build {branch}: {main_github_sha}\n\n"
+            "<details><summary>Build size details</summary><p>\n\n"
+            f"{branch}: {main_github_sha} ydbd build size:\n"
+            f"  - binary size **{format_number(main_size_bytes)}** Bytes \n"
+            f"  - stripped binary size **{format_number(main_size_stripped_bytes)}** Bytes\n\n"
+            f"merge: {current_pr_commit_sha} ydbd build size:\n"
+            f"  - binary size **{format_number(current_size_bytes)}** Bytes \n"
+            f"  - stripped binary size **{format_number(current_size_stripped_bytes)}** Bytes\n\n"
+            "[ydbd size dashboard](https://datalens.yandex/cu6hzmpaki700)\n\n"
+            "</p></details>"
+        )
+        print(f"{color};;;{comment}")
+    else:
+        print(f'Error: Cant get build data: {branch}_sizes_result = {main_sizes_result}, current_sizes_result = {current_sizes_result}')   
+    
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/scripts/get_current_build_size.py b/.github/scripts/get_current_build_size.py
new file mode 100755
index 0000000000..658598478f
--- /dev/null
+++ b/.github/scripts/get_current_build_size.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import configparser
+import os
+import subprocess
+
+
+dir = os.path.dirname(__file__)
+config = configparser.ConfigParser()
+config_file_path = f"{dir}/../config/ydb_qa_db.ini"
+config.read(config_file_path)
+
+YDBD_PATH = config["YDBD"]["YDBD_PATH"]
+
+
+def get_build_size():
+
+    if not os.path.exists(YDBD_PATH):
+        # can be possible due to incremental builds and ydbd itself is not affected by changes
+        print("Error: {} not exists, skipping".format(YDBD_PATH))
+        return 0
+
+    binary_size_bytes = subprocess.check_output(
+        ["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)]
+    )
+    binary_size_stripped_bytes = subprocess.check_output(
+        ["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)]
+    )
+
+    size_stripped_bytes = int(binary_size_stripped_bytes.decode("utf-8"))
+    size_bytes = int(binary_size_bytes.decode("utf-8"))
+    if binary_size_bytes and binary_size_stripped_bytes:
+        return {"size_bytes": size_bytes, "size_stripped_bytes": size_stripped_bytes}
+    else:
+        print(f"Error: Cant get build size")
+        return 1
+
+
+if __name__ == "__main__":
+    get_build_size()
diff --git a/.github/scripts/get_main_build_size.py b/.github/scripts/get_main_build_size.py
new file mode 100755
index 0000000000..324329f6cf
--- /dev/null
+++ b/.github/scripts/get_main_build_size.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import configparser
+import os
+import ydb
+
+
+dir = os.path.dirname(__file__)
+config = configparser.ConfigParser()
+config_file_path = f"{dir}/../config/ydb_qa_db.ini"
+config.read(config_file_path)
+
+build_preset = os.environ.get("build_preset")
+branch = os.environ.get("branch_to_compare")
+
+DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
+DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
+
+
+def get_build_size():
+    if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ:
+        print(
+            "Error: Env variable CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping"
+        )
+        return 0
+    else:
+        # Do not set up 'real' variable from gh workflows because it interfere with ydb tests
+        # So, set up it locally
+        os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = os.environ[
+            "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"
+        ]
+
+    sql = f"""
+    --!syntax_v1
+    select git_commit_time,github_sha,size_bytes,size_stripped_bytes,build_preset from binary_size 
+    where 
+    github_workflow like "Postcommit%" and 
+    github_ref_name="{branch}" and 
+    build_preset="{build_preset}"
+    order by git_commit_time desc
+    limit 1;    
+    """
+
+    with ydb.Driver(
+        endpoint=DATABASE_ENDPOINT,
+        database=DATABASE_PATH,
+        credentials=ydb.credentials_from_env_variables(),
+    ) as driver:
+        driver.wait(timeout=10, fail_fast=True)
+        session = ydb.retry_operation_sync(
+            lambda: driver.table_client.session().create()
+        )
+        with session.transaction() as transaction:
+            result = transaction.execute(sql, commit_tx=True)
+            if result[0].rows:
+                for row in result[0].rows:
+                    main_data = {}
+                    for field in row:
+                        main_data[field] = (
+                            row[field]
+                            if type(row[field]) != bytes
+                            else row[field].decode("utf-8")
+                        )
+            else:
+                print(
+                    f"Error: Cant get binary size in db with params: github_workflow like 'Postcommit%', github_ref_name='{branch}', build_preset='{build_preset}'"
+                )
+                return 0
+
+        return {
+            "github_sha": main_data["github_sha"],
+            "git_commit_time": str(main_data["git_commit_time"]),
+            "size_bytes": str(main_data["size_bytes"]),
+            "size_stripped_bytes": str(main_data["size_stripped_bytes"]),
+        }
+
+
+if __name__ == "__main__":
+    get_build_size()
diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py
index 73496cd345..de9dc171a8 100755
--- a/.github/scripts/send_build_stats.py
+++ b/.github/scripts/send_build_stats.py
@@ -1,15 +1,20 @@
 #!/usr/bin/env python3
 
+import configparser
 import datetime
 import os
 import ydb
 import uuid
 import subprocess
 
+dir = os.path.dirname(__file__)
+config = configparser.ConfigParser()
+config_file_path = f"{dir}/../config/ydb_qa_db.ini"
+config.read(config_file_path)
 
-YDBD_PATH = "ydb/apps/ydbd/ydbd"
-DATABASE_PATH = "/ru-central1/b1ggceeul2pkher8vhb6/etnvsjbk7kh1jc6bbfi8"
-DATABASE_ENDPOINT = "grpcs://lb.etnvsjbk7kh1jc6bbfi8.ydb.mdb.yandexcloud.net:2135"
+YDBD_PATH = config["YDBD"]["YDBD_PATH"]
+DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
+DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
 
 FROM_ENV_COLUMNS = [
     "github_head_ref",
@@ -108,7 +113,7 @@ VALUES
             )
 
             build_preset = os.environ.get("build_preset", None)
-            github_sha = os.environ.get("GITHUB_SHA", None)
+            github_sha = os.environ.get("commit_git_sha", None)
 
             if github_sha is not None:
                 git_commit_time_bytes = subprocess.check_output(
@@ -140,7 +145,10 @@ VALUES
             for column in FROM_ENV_COLUMNS:
                 value = os.environ.get(column.upper(), None)
                 parameters["$" + column] = sanitize_str(value)
-
+            
+            #workaround for https://github.com/ydb-platform/ydb/issues/5294
+            parameters["$github_sha"] = sanitize_str(github_sha)
+            
             print("Executing query:\n{}".format(text_query))
             print("With parameters:")
             for k, v in parameters.items():
diff --git a/.github/scripts/tests/generate-summary.py b/.github/scripts/tests/generate-summary.py
index b7b94004cf..aef7074652 100755
--- a/.github/scripts/tests/generate-summary.py
+++ b/.github/scripts/tests/generate-summary.py
@@ -202,6 +202,8 @@ class TestSummary:
         if add_footnote:
             result.append("")
             result.append(f"[^1]: All mute rules are defined [here]({footnote_url}).")
+            
+        result.append("")
         return result