Support linkage in build_ya (second version) (#4752)

3 files changed, 39 insertions, 26 deletions

diff --git a/.github/actions/build_and_test_ya/action.yml b/.github/actions/build_and_test_ya/action.yml
index 26c46d2df8..8c852ad4e0 100644
--- a/.github/actions/build_and_test_ya/action.yml
+++ b/.github/actions/build_and_test_ya/action.yml
@@ -77,6 +77,20 @@ runs:
         folder_prefix: ya-
         build_preset: ${{ inputs.build_preset }}
 
+    - name: Generate ya.make with affected test suites and modules list
+      if: inputs.increment == 'true'
+      uses: ./.github/actions/graph_compare
+ 
+    - name: Change target in case of incremental (tests)
+      id: test_run_choice
+      shell: bash
+      run: |
+        if [ "${{ inputs.increment }}" == "true" ]; then
+          echo "target=." >> $GITHUB_OUTPUT
+        else
+          echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
+        fi
+
     - name: Build
       uses: ./.github/actions/build_ya
       id: build
@@ -89,22 +103,6 @@ runs:
         bazel_remote_password: ${{ inputs.put_build_results_to_cache &&  fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
         link_threads: ${{ inputs.link_threads }}
 
-    - name: Generate ya.make with affected test suites list
-      if: inputs.run_tests == 'true' && inputs.increment == 'true'
-      uses: ./.github/actions/graph_compare
- 
-    - name: Check if there's a list of tests to run
-      id: test_run_choice
-      shell: bash
-      run: |
-        if [ -f ya.make ];then
-          echo "target='.'" >> $GITHUB_OUTPUT
-          echo "Listed test targets: "
-          cat ya.make
-        else
-          echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
-        fi
-
     - name: Run tests
       uses: ./.github/actions/test_ya
       if: ${{ inputs.run_tests == 'true' &&  (steps.build.outputs.success == 'true' || inputs.run_tests_if_build_fails == 'true') }}
diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml
index 7f48dd4b46..3eb4154e0c 100644
--- a/.github/actions/test_ya/action.yml
+++ b/.github/actions/test_ya/action.yml
@@ -191,13 +191,16 @@ runs:
             ;;
         esac
         
+        # Also build targets which are not in tests' dependencies
+        params+=(--no-strip-idle-build-results)
+
         echo "::debug::get version"
         ./ya --version
         
         echo "Tests are running..." | .github/scripts/tests/comment-pr.py
 
         ./ya test ${{ inputs.build_target }} "${params[@]}" \
-          --stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" -DCONSISTENT_DEBUG \
+          --stat --log-file "$LOG_DIR/ya_log2.txt" --evlog-file "$LOG_DIR/ya_evlog2.jsonl" -DCONSISTENT_DEBUG \
           --no-dir-outputs \
           --junit "$JUNIT_REPORT_XML" --output "$OUT_DIR" || (
             RC=$?
diff --git a/.github/scripts/graph_compare.sh b/.github/scripts/graph_compare.sh
index 89ebd65fd5..0e873fff41 100755
--- a/.github/scripts/graph_compare.sh
+++ b/.github/scripts/graph_compare.sh
@@ -1,21 +1,21 @@
 
 # Compares build graphs for two given refs in the current directory git repo
-# Creates ya.make in the current directory listing affected ydb test suites
+# Creates ya.make in the current directory listing affected ydb targets
 # Parameters: base_commit_sha head_commit_sha
 
-set -e
+set -ex
 
 workdir=$(mktemp -d)
 echo Workdir: $workdir
 echo Checkout base commit...
 git checkout $1
 echo Build graph for base commit...
-./ya make -Gj0 -ttt ydb --build release -k --cache-tests | jq '.graph[] | select( ."node-type"=="test")' > $workdir/graph_base
+./ya make -Gj0 -ttt ydb --build release -k --cache-tests --no-strip-idle-build-results | jq '.graph[]' > $workdir/graph_base
 
 echo Checkout head commit...
 git checkout $2
 echo Build graph for head commit...
-./ya make -Gj0 -ttt ydb --build release -k --cache-tests | jq '.graph[] | select( ."node-type"=="test")' > $workdir/graph_head
+./ya make -Gj0 -ttt ydb --build release -k --cache-tests --no-strip-idle-build-results | jq '.graph[]' > $workdir/graph_head
 
 echo Generate lists of uids for base and head...
 cat $workdir/graph_base | jq '.uid' > $workdir/uid_base
@@ -24,8 +24,11 @@ cat $workdir/graph_head | jq '.uid' > $workdir/uid_head
 echo Create a list of changed uids in the head graph...
 (cat $workdir/uid_head;(cat $workdir/uid_base;cat $workdir/uid_head) | sort | uniq -u) | sort | uniq -d > $workdir/uids_new
 
+echo Create ya.make
+echo "" > ya.make
+
 echo Generate list of test shard names from the head graph based on the list of uids...
-cat $workdir/graph_head | jq -r --slurpfile uids $workdir/uids_new 'select( any( .uid; .==$uids[] )) | .kv.path' | sort | uniq > $workdir/testsuites
+cat $workdir/graph_head | jq -r --slurpfile uids $workdir/uids_new 'select( ."node-type"=="test") | select( any( .uid; .==$uids[] )) | .kv.path' | sort | uniq > $workdir/testsuites
 
 echo Number of test suites:
 cat $workdir/testsuites | wc -l
@@ -33,8 +36,17 @@ cat $workdir/testsuites | wc -l
 echo Removing test suite name from the list to get target names...
 sed -E 's/\/[^/]*$//g;/^null$/d' $workdir/testsuites > $workdir/ts2
 
-echo Generating temp ya.make with recurses to all required tests...
-cat $workdir/ts2 | (echo 'RECURSE_FOR_TESTS(';cat;echo ')') > ya.make
+echo Append into ya.make RECURSE_FOR_TESTS to all required tests...
+cat $workdir/ts2 | (echo 'RECURSE_FOR_TESTS(';cat;echo ')') >> ya.make
+
+echo Generate list of module names from the head graph based on the list of uids...
+cat $workdir/graph_head | jq -r --slurpfile uids $workdir/uids_new 'select( ."target_properties"."module_type" != null) | select( any( .uid; .==$uids[] )) | .target_properties.module_dir' | sort | uniq > $workdir/modules
+
+echo Number of modules:
+cat $workdir/modules | wc -l
+
+echo Append into ya.make RECURSE to all required modules...
+cat $workdir/modules | (echo 'RECURSE(';cat;echo ')') >> ya.make
 
-# echo Running ya test...
-# ./ya make -A -R --build relwithdebinfo .
\ No newline at end of file
+echo "ya.make content:"
+cat ya.make