diff options
author | Kirill Rysin <35688753+naspirato@users.noreply.github.com> | 2024-06-18 18:25:51 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-06-18 18:25:51 +0200 |
commit | 949521ce54af90a84bd2d4e1daec96e8e884e156 (patch) | |
tree | 14d54001922493374f15a30833181bcd1f31096b | |
parent | 3833ea49d4d7749bdd08fe764aff8bb4ef76d7a7 (diff) | |
download | ydb-949521ce54af90a84bd2d4e1daec96e8e884e156.tar.gz |
5295 add precommit check for ydbd size add diff in pr (#5597)
-rw-r--r-- | .github/actions/build_and_test_ya/action.yml | 38 | ||||
-rw-r--r-- | .github/config/ydb_qa_db.ini | 5 | ||||
-rwxr-xr-x | .github/scripts/get_build_diff.py | 76 | ||||
-rwxr-xr-x | .github/scripts/get_current_build_size.py | 40 | ||||
-rwxr-xr-x | .github/scripts/get_main_build_size.py | 79 | ||||
-rwxr-xr-x | .github/scripts/send_build_stats.py | 18 | ||||
-rwxr-xr-x | .github/scripts/tests/generate-summary.py | 2 |
7 files changed, 252 insertions, 6 deletions
diff --git a/.github/actions/build_and_test_ya/action.yml b/.github/actions/build_and_test_ya/action.yml index 5749b7178e..6f2d321125 100644 --- a/.github/actions/build_and_test_ya/action.yml +++ b/.github/actions/build_and_test_ya/action.yml @@ -132,7 +132,8 @@ runs: link_threads: ${{ inputs.link_threads }} additional_ya_make_args: ${{ inputs.additional_ya_make_args }} test_threads: ${{ inputs.test_threads }} - + + - name: Notify about failed build if: ${{ steps.build.outputs.success != 'true' && inputs.run_tests == 'true' && inputs.run_tests_if_build_fails == 'false' }} shell: bash @@ -152,9 +153,44 @@ runs: run: | set -x export build_preset="${{ inputs.build_preset }}" + export commit_git_sha="$(git rev-parse HEAD)" + python3 -m pip install ydb ydb[yc] python3 .github/scripts/send_build_stats.py + - name: show_build_size_diff + shell: bash + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -x + export build_preset="${{ inputs.build_preset }}" + export branch_to_compare="$GITHUB_REF_NAME" + export yellow_treshold=102400 + export red_treshold=2097152 + export commit_git_sha="$(git rev-parse HEAD)" + + python3 -m pip install ydb ydb[yc] humanize + + get_sizes_comment_script=.github/scripts/get_build_diff.py + comment_raw=`$get_sizes_comment_script` + + IFS=';;;' + read -ra comment_arr <<< "$comment_raw" + + printf "$comment" + if [[ ${comment_raw} != "Error"* ]];then + color=${comment_arr[0]} + replace=$color";;;" + comment=${comment_raw/$replace/""} + + printf "$comment" | .github/scripts/tests/comment-pr.py --color $color + + else + echo "Skipped build size difference, comment_raw = ${comment_raw}" + fi + - name: comment-if-cancel shell: bash if: cancelled() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') diff --git a/.github/config/ydb_qa_db.ini b/.github/config/ydb_qa_db.ini new file mode 100644 index 0000000000..f15bb526f8 --- /dev/null +++ b/.github/config/ydb_qa_db.ini @@ -0,0 +1,5 @@ +[QA_DB] +DATABASE_PATH = /ru-central1/b1ggceeul2pkher8vhb6/etnvsjbk7kh1jc6bbfi8 +DATABASE_ENDPOINT = grpcs://lb.etnvsjbk7kh1jc6bbfi8.ydb.mdb.yandexcloud.net:2135 +[YDBD] +YDBD_PATH = ydb/apps/ydbd/ydbd
\ No newline at end of file diff --git a/.github/scripts/get_build_diff.py b/.github/scripts/get_build_diff.py new file mode 100755 index 0000000000..30ba3f46c4 --- /dev/null +++ b/.github/scripts/get_build_diff.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +from decimal import Decimal, ROUND_HALF_UP +import get_current_build_size +import get_main_build_size +import humanize +import os + + +# Форматирование числа +def format_number(num): + return humanize.intcomma(num).replace(",", " ") + + +def bytes_to_human_iec(num): + return humanize.naturalsize(num, binary=True) + + +def main(): + + yellow_treshold = int(os.environ.get("yellow_treshold")) + red_treshold = int(os.environ.get("red_treshold")) + + branch = os.environ.get("branch_to_compare") + current_pr_commit_sha = os.environ.get("commit_git_sha") + + current_sizes_result = get_current_build_size.get_build_size() + main_sizes_result = get_main_build_size.get_build_size() + + if main_sizes_result and current_sizes_result: + main_github_sha = main_sizes_result["github_sha"] + main_size_bytes = int(main_sizes_result["size_bytes"]) + main_size_stripped_bytes = int(main_sizes_result["size_stripped_bytes"]) + + current_size_bytes = int(current_sizes_result["size_bytes"]) + current_size_stripped_bytes = int(current_sizes_result["size_stripped_bytes"]) + + bytes_diff = current_size_bytes - main_size_bytes + diff_perc = Decimal(bytes_diff * 100 / main_size_bytes).quantize( + Decimal(".001"), rounding=ROUND_HALF_UP + ) + + human_readable_size = bytes_to_human_iec(current_size_bytes) + human_readable_size_diff = bytes_to_human_iec(bytes_diff) + + if bytes_diff >= 0: + sign = "+" + if bytes_diff >= red_treshold: + color = "red" + elif bytes_diff >= yellow_treshold: + color = "yellow" + else: + color = "green" + else: + sign = "" + color = "green" + + comment = ( + f"merge: {current_pr_commit_sha} ydbd size {human_readable_size} **{sign}{human_readable_size_diff} {sign}{diff_perc}%%** vs build {branch}: {main_github_sha}\n\n" + "<details><summary>Build size details</summary><p>\n\n" + f"{branch}: {main_github_sha} ydbd build size:\n" + f" - binary size **{format_number(main_size_bytes)}** Bytes \n" + f" - stripped binary size **{format_number(main_size_stripped_bytes)}** Bytes\n\n" + f"merge: {current_pr_commit_sha} ydbd build size:\n" + f" - binary size **{format_number(current_size_bytes)}** Bytes \n" + f" - stripped binary size **{format_number(current_size_stripped_bytes)}** Bytes\n\n" + "[ydbd size dashboard](https://datalens.yandex/cu6hzmpaki700)\n\n" + "</p></details>" + ) + print(f"{color};;;{comment}") + else: + print(f'Error: Cant get build data: {branch}_sizes_result = {main_sizes_result}, current_sizes_result = {current_sizes_result}') + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/get_current_build_size.py b/.github/scripts/get_current_build_size.py new file mode 100755 index 0000000000..658598478f --- /dev/null +++ b/.github/scripts/get_current_build_size.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import configparser +import os +import subprocess + + +dir = os.path.dirname(__file__) +config = configparser.ConfigParser() +config_file_path = f"{dir}/../config/ydb_qa_db.ini" +config.read(config_file_path) + +YDBD_PATH = config["YDBD"]["YDBD_PATH"] + + +def get_build_size(): + + if not os.path.exists(YDBD_PATH): + # can be possible due to incremental builds and ydbd itself is not affected by changes + print("Error: {} not exists, skipping".format(YDBD_PATH)) + return 0 + + binary_size_bytes = subprocess.check_output( + ["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)] + ) + binary_size_stripped_bytes = subprocess.check_output( + ["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)] + ) + + size_stripped_bytes = int(binary_size_stripped_bytes.decode("utf-8")) + size_bytes = int(binary_size_bytes.decode("utf-8")) + if binary_size_bytes and binary_size_stripped_bytes: + return {"size_bytes": size_bytes, "size_stripped_bytes": size_stripped_bytes} + else: + print(f"Error: Cant get build size") + return 1 + + +if __name__ == "__main__": + get_build_size() diff --git a/.github/scripts/get_main_build_size.py b/.github/scripts/get_main_build_size.py new file mode 100755 index 0000000000..324329f6cf --- /dev/null +++ b/.github/scripts/get_main_build_size.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import configparser +import os +import ydb + + +dir = os.path.dirname(__file__) +config = configparser.ConfigParser() +config_file_path = f"{dir}/../config/ydb_qa_db.ini" +config.read(config_file_path) + +build_preset = os.environ.get("build_preset") +branch = os.environ.get("branch_to_compare") + +DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"] +DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"] + + +def get_build_size(): + if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ: + print( + "Error: Env variable CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping" + ) + return 0 + else: + # Do not set up 'real' variable from gh workflows because it interfere with ydb tests + # So, set up it locally + os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = os.environ[ + "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" + ] + + sql = f""" + --!syntax_v1 + select git_commit_time,github_sha,size_bytes,size_stripped_bytes,build_preset from binary_size + where + github_workflow like "Postcommit%" and + github_ref_name="{branch}" and + build_preset="{build_preset}" + order by git_commit_time desc + limit 1; + """ + + with ydb.Driver( + endpoint=DATABASE_ENDPOINT, + database=DATABASE_PATH, + credentials=ydb.credentials_from_env_variables(), + ) as driver: + driver.wait(timeout=10, fail_fast=True) + session = ydb.retry_operation_sync( + lambda: driver.table_client.session().create() + ) + with session.transaction() as transaction: + result = transaction.execute(sql, commit_tx=True) + if result[0].rows: + for row in result[0].rows: + main_data = {} + for field in row: + main_data[field] = ( + row[field] + if type(row[field]) != bytes + else row[field].decode("utf-8") + ) + else: + print( + f"Error: Cant get binary size in db with params: github_workflow like 'Postcommit%', github_ref_name='{branch}', build_preset='{build_preset}'" + ) + return 0 + + return { + "github_sha": main_data["github_sha"], + "git_commit_time": str(main_data["git_commit_time"]), + "size_bytes": str(main_data["size_bytes"]), + "size_stripped_bytes": str(main_data["size_stripped_bytes"]), + } + + +if __name__ == "__main__": + get_build_size() diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index 73496cd345..de9dc171a8 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -1,15 +1,20 @@ #!/usr/bin/env python3 +import configparser import datetime import os import ydb import uuid import subprocess +dir = os.path.dirname(__file__) +config = configparser.ConfigParser() +config_file_path = f"{dir}/../config/ydb_qa_db.ini" +config.read(config_file_path) -YDBD_PATH = "ydb/apps/ydbd/ydbd" -DATABASE_PATH = "/ru-central1/b1ggceeul2pkher8vhb6/etnvsjbk7kh1jc6bbfi8" -DATABASE_ENDPOINT = "grpcs://lb.etnvsjbk7kh1jc6bbfi8.ydb.mdb.yandexcloud.net:2135" +YDBD_PATH = config["YDBD"]["YDBD_PATH"] +DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"] +DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"] FROM_ENV_COLUMNS = [ "github_head_ref", @@ -108,7 +113,7 @@ VALUES ) build_preset = os.environ.get("build_preset", None) - github_sha = os.environ.get("GITHUB_SHA", None) + github_sha = os.environ.get("commit_git_sha", None) if github_sha is not None: git_commit_time_bytes = subprocess.check_output( @@ -140,7 +145,10 @@ VALUES for column in FROM_ENV_COLUMNS: value = os.environ.get(column.upper(), None) parameters["$" + column] = sanitize_str(value) - + + #workaround for https://github.com/ydb-platform/ydb/issues/5294 + parameters["$github_sha"] = sanitize_str(github_sha) + print("Executing query:\n{}".format(text_query)) print("With parameters:") for k, v in parameters.items(): diff --git a/.github/scripts/tests/generate-summary.py b/.github/scripts/tests/generate-summary.py index b7b94004cf..aef7074652 100755 --- a/.github/scripts/tests/generate-summary.py +++ b/.github/scripts/tests/generate-summary.py @@ -202,6 +202,8 @@ class TestSummary: if add_footnote: result.append("") result.append(f"[^1]: All mute rules are defined [here]({footnote_url}).") + + result.append("") return result |