Test bloat for every PR (#9948)

5 files changed, 112 insertions, 13 deletions

diff --git a/.github/actions/build_analytics/action.yml b/.github/actions/build_analytics/action.yml
index c4a79e6dda..e8018eec64 100644
--- a/.github/actions/build_analytics/action.yml
+++ b/.github/actions/build_analytics/action.yml
@@ -45,7 +45,7 @@ runs:
       run: |
         set -ex
         s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "ya_bloat_html/" "$S3_BUCKET_PATH/ya_bloat_html/"
-        echo "[ya bloat tool]($S3_URL_PREFIX/ya_bloat_html/index.html) (better use Safari, because it is too large for Chrome)" >> $GITHUB_STEP_SUMMARY
+        echo "[ya bloat tool]($S3_URL_PREFIX/ya_bloat_html/tree_map.html) (better use Safari, because it is too large for Chrome)" >> $GITHUB_STEP_SUMMARY
 
         s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "template_bloat.by_size.txt" "$S3_BUCKET_PATH/template_bloat.by_size.txt"
         TEMPLATE_BLOAT_BY_SIZE_URL=$S3_URL_PREFIX/template_bloat.by_size.txt
@@ -54,13 +54,13 @@ runs:
         TEMPLATE_BLOAT_BY_COUNT_URL=$S3_URL_PREFIX/template_bloat.by_count.txt
 
         s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "template_bloat_html/" "$S3_BUCKET_PATH/template_bloat_html/"
-        TEMPLATE_BLOAT_TREEMAP_URL=$S3_URL_PREFIX/template_bloat_html/index.html
+        TEMPLATE_BLOAT_TREEMAP_URL=$S3_URL_PREFIX/template_bloat_html/tree_map.html
 
         echo "[template bloat]($TEMPLATE_BLOAT_TREEMAP_URL) ([sorted by size]($TEMPLATE_BLOAT_BY_SIZE_URL), [sorted by count]($TEMPLATE_BLOAT_BY_COUNT_URL))" >> $GITHUB_STEP_SUMMARY
 
         s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_cpp_impact/" "$S3_BUCKET_PATH/html_cpp_impact/"
-        echo "[cpp compilation time]($S3_URL_PREFIX/html_cpp_impact/index.html)" >> $GITHUB_STEP_SUMMARY
+        echo "[cpp compilation time]($S3_URL_PREFIX/html_cpp_impact/tree_map.html)" >> $GITHUB_STEP_SUMMARY
         
         s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_headers_impact/" "$S3_BUCKET_PATH/html_headers_impact/"
-        echo "[headers impact]($S3_URL_PREFIX/html_headers_impact/index.html)" >> $GITHUB_STEP_SUMMARY
+        echo "[headers impact]($S3_URL_PREFIX/html_headers_impact/tree_map.html)" >> $GITHUB_STEP_SUMMARY
 
diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml
index c3ec743968..354105cb42 100644
--- a/.github/actions/test_ya/action.yml
+++ b/.github/actions/test_ya/action.yml
@@ -122,7 +122,7 @@ runs:
         echo "SUMMARY_LINKS=$PUBLIC_DIR/summary_links.txt" >> $GITHUB_ENV
         echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV
 
-        python3 -m pip install ydb ydb[yc] codeowners
+        python3 -m pip install ydb ydb[yc] codeowners humanize junitparser
 
         if [ ${{ inputs.testman_token }} ]; then
           TESTMO_PROXY_ADDR=127.0.0.1:8888
@@ -399,6 +399,9 @@ runs:
           ya_dir=$(pwd)
           (cd $CURRENT_PUBLIC_DIR && $ya_dir/ya analyze-make timeline --evlog ya_evlog.jsonl)
 
+          # generate test_bloat
+          ./ydb/ci/build_bloat/test_bloat.py --junit $CURRENT_JUNIT_XML_PATH --output_dir $CURRENT_PUBLIC_DIR/test_bloat || true
+
           if [ $RC -ne 0 ]; then
             echo "ya make returned $RC, build failed"
             echo "status=failed" >> $GITHUB_OUTPUT
@@ -453,7 +456,7 @@ runs:
               "Tests" $CURRENT_PUBLIC_DIR/ya-test.html "$CURRENT_JUNIT_XML_PATH"
           fi
 
-          s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"
+          s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-mime-magic --guess-mime-type --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"
 
           if [ "${{ inputs.run_tests }}" = "true" ]; then
             cat summary_text.txt | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py --color `cat summary_color.txt`
@@ -595,7 +598,7 @@ runs:
         echo "::group::s3-sync"
         .github/scripts/Indexer/indexer.py -r "$PUBLIC_DIR/"
         echo "00 [Artifacts](${PUBLIC_DIR_URL}/index.html)" >> $SUMMARY_LINKS
-        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"
+        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-mime-magic --guess-mime-type --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
         echo "::endgroup::"
 
@@ -613,7 +616,6 @@ runs:
         export build_preset="${{ inputs.build_preset }}" 
         export commit_git_sha="$(git rev-parse HEAD)"
 
-        python3 -m pip install ydb ydb[yc]
         python3 .github/scripts/send_build_stats.py
 
     - name: show_build_size_diff
@@ -630,8 +632,6 @@ runs:
         export red_treshold=2097152 
         export commit_git_sha="$(git rev-parse HEAD)"
 
-        python3 -m pip install ydb ydb[yc] humanize
-
         get_sizes_comment_script=.github/scripts/get_build_diff.py
         comment_raw=`$get_sizes_comment_script`
         
diff --git a/ydb/ci/build_bloat/html/index.html b/ydb/ci/build_bloat/html/tree_map.html
index 1c352ebaf5..1c352ebaf5 100644
--- a/ydb/ci/build_bloat/html/index.html
+++ b/ydb/ci/build_bloat/html/tree_map.html
diff --git a/ydb/ci/build_bloat/test_bloat.py b/ydb/ci/build_bloat/test_bloat.py
new file mode 100755
index 0000000000..078b77dc9f
--- /dev/null
+++ b/ydb/ci/build_bloat/test_bloat.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+# Tool used to transform junit report. Performs the following:
+# - adds classname with relative path to test in 'testcase' node
+# - add 'url:logsdir' and other links with in 'testcase' node
+# - mutes tests
+
+import argparse
+
+from junitparser import JUnitXml
+
+import tree_map
+
+THRESHOLD = 5  # sec
+
+
+def generate(junit_path, output_dir):
+    junit = JUnitXml.fromfile(junit_path)
+
+    tree_paths = []
+    tree_paths.append([["/", "dir", 0]])
+
+    for suite in junit:
+        for case in suite:
+            test_name = case.name
+            duration = case.time
+            
+            test_name = test_name.split("[")[0]
+
+            root_name = "/"
+            path = [root_name] + suite.name.split("/") + [test_name]
+            path_with_info = [[chunk, "dir", 0] for chunk in path]
+            path_with_info[-1][1] = "testcase"
+            path_with_info[-1][2] = duration
+            path_with_info[-2][1] = "testsuite"
+            tree_paths.append(path_with_info)
+
+
+    types = [
+        ("dir", "Directory", "#66C2A5"),
+        ("testsuite", "Test Suite", "#8DA0CB"),
+        ("testcase", "Test Case", "#FC8D62"),
+    ]
+    tree_map.generate_tree_map_html(
+        output_dir, 
+        tree_paths, 
+        unit_name="sec", 
+        factor=1.0, 
+        types=types, 
+        threshold=THRESHOLD, 
+        fix_size_threshold=True
+    )
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="""A tool for analyzing tests time\n
+
+To use it run ya make with '--junit <path>' and junit file will be generated"""
+    )
+    parser.add_argument(
+        "-j",
+        "--junit",
+        required=True,
+        help="Path to junit.xml",
+    )
+    parser.add_argument(
+        "-o",
+        "--output_dir",
+        required=False,
+        default="test_bloat",
+        help="Output path for treemap view of compilation times",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    generate(args.junit, args.output_dir)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/ydb/ci/build_bloat/tree_map.py b/ydb/ci/build_bloat/tree_map.py
index fc02b1ca6d..818667806f 100755
--- a/ydb/ci/build_bloat/tree_map.py
+++ b/ydb/ci/build_bloat/tree_map.py
@@ -38,6 +38,21 @@ def _propogate_size(tree):
         tree["size"] += _propogate_size(child)
     return tree["size"]
 
+def _remove_less_then_threshold(tree, threshold, fix_size_threshold):
+    new_children = []
+    new_size = 0
+    self_size = tree["size"]
+    for child in tree.get("children", []):
+        self_size -= child["size"]
+        _remove_less_then_threshold(child, threshold, fix_size_threshold)
+        if child["size"] >= threshold:
+            new_children.append(child)
+            new_size += child["size"]
+
+    tree["children"] = new_children
+    if fix_size_threshold:
+        tree["size"] = new_size + self_size
+
 def _intify_size(tree):
     for child in tree.get("children", []):
         _intify_size(child)
@@ -49,22 +64,23 @@ def _enrich_names_with_units(tree, unit_name, factor):
 
     tree["name"] = tree["name"] + ", {:_} {}".format(int(tree["size"]*factor), unit_name)
 
-def _build_tree_map(paths_to_add, unit_name, factor):
+def _build_tree_map(paths_to_add, unit_name, factor, threshold, fix_size_threshold):
     tree = {}
     for path in paths_to_add:
         _add_to_tree(tree, path)
     _children_to_list(tree)
     _propogate_size(tree)
+    _remove_less_then_threshold(tree, threshold, fix_size_threshold)
     _intify_size(tree)
     _enrich_names_with_units(tree, unit_name, factor)
     return tree
 
 
-def generate_tree_map_html(output_dir: str, tree_paths: list[tuple[str, str, int]], unit_name: str, factor: float, types: list[tuple[str, str, str]]):
+def generate_tree_map_html(output_dir: str, tree_paths: list[tuple[str, str, int]], unit_name: str, factor: float, types: list[tuple[str, str, str]], threshold=0, fix_size_threshold=False):
     current_script_dir = os.path.dirname(os.path.realpath(__file__))
     html_dir = os.path.join(current_script_dir, "html")
 
-    tree = _build_tree_map(tree_paths, unit_name, factor)
+    tree = _build_tree_map(tree_paths, unit_name, factor, threshold, fix_size_threshold)
 
     env = Environment(loader=FileSystemLoader(html_dir), undefined=StrictUndefined)
     file_names = os.listdir(html_dir)