bring workflows from main

3 files changed, 236 insertions, 0 deletions

diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
new file mode 100644
index 0000000000..6b754d2b7a
--- /dev/null
+++ b/.github/actions/build/action.yml
@@ -0,0 +1,46 @@
+name: Build
+description: Build YDB
+inputs:
+  sanitizer:
+    required: false
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+  - name: Configure for sanitizer
+    shell: bash
+    if: inputs.sanitizer
+    run: |
+      mkdir -p ../build
+      patch -p1 < ydb/deploy/patches/0001-sanitizer-build.patch
+      cd ../build
+      rm -rf *
+      cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
+        -DCMAKE_CXX_FLAGS="-fsanitize=${{ inputs.sanitizer }} -g -fno-omit-frame-pointer" \
+        -DCMAKE_C_FLAGS="-fsanitize=${{ inputs.sanitizer }} -g -fno-omit-frame-pointer" \
+        -DCMAKE_EXE_LINKER_FLAGS="-rdynamic" \
+        ../ydb
+  - name: Configure
+    shell: bash
+    if: ${{!inputs.sanitizer}}
+    run: |
+      mkdir -p ../build
+      cd ../build
+      rm -rf *
+      cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+        -DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
+        -DCMAKE_CXX_FLAGS="-g" \
+        -DCMAKE_C_FLAGS="-g" \
+        -DCMAKE_EXE_LINKER_FLAGS="-rdynamic" \
+        ../ydb
+  - name: Build
+    shell: bash
+    run: |
+      ccache -z
+      cd ../build
+      ninja
+      ccache -s
diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
new file mode 100644
index 0000000000..0a3c848c84
--- /dev/null
+++ b/.github/actions/test/action.yml
@@ -0,0 +1,128 @@
+name: build-and-test
+description: Build YDB and run Tests
+inputs:
+  log_suffix:
+    required: true
+    type: string
+  test_label_regexp:
+    required: false
+    type: string
+  aws_key_id:
+    required: true
+    type: string
+  aws_key_value: 
+    required: true
+    type: string
+  testman_token:
+    required: false
+    type: string
+  testman_url:
+    required: false
+    type: string
+  testman_project_id:
+    required: false
+    type: string
+  aws_bucket:
+    required: true
+    type: string
+  aws_endpoint:
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+  - name: Init
+    id: init
+    shell: bash
+    run: |
+      mkdir -p artifacts tmp test_reports
+      rm -rf artifacts/* tmp/* test_reports/*
+      echo "WORKDIR=$(pwd)" >> $GITHUB_ENV
+      echo "TESTREPDIR=$(pwd)/test_reports" >> $GITHUB_ENV
+      echo "TESTMO_TOKEN=${{inputs.testman_token}}" >> $GITHUB_ENV
+      echo "TESTMO_URL=${{inputs.testman_url}}" >> $GITHUB_ENV
+      echo "logfilename=${{inputs.log_suffix}}-ctest-stdout.gz" >> $GITHUB_OUTPUT
+  - name: Install Node required for Testmo CLI
+    uses: actions/setup-node@v3
+    with:
+      node-version: 19
+  - name: Install Testmo CLI
+    shell: bash
+    run: npm install -g @testmo/testmo-cli
+  - name: Test history run create
+    id: th
+    if: inputs.testman_token
+    shell: bash
+    env:
+      PR_NUMBER: ${{ github.event.number }}
+    run: |
+      RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+      BRANCH_TAG="$GITHUB_REF_NAME"
+
+      case $GITHUB_EVENT_NAME in
+        workflow_dispatch)
+          TESTMO_RUN_NAME="${{ github.run_id }} manual"
+          EXTRA_TAG="manual"
+          ;;
+        pull_request | pull_request_target)
+          TESTMO_RUN_NAME="${{ github.run_id }} PR #${PR_NUMBER}"
+          EXTRA_TAG="pr"
+          BRANCH_TAG=""
+          ;;
+        schedule)
+          TESTMO_RUN_NAME="${{ github.run_id }} schedule"
+          EXTRA_TAG="schedule"
+          ;;
+        *)
+          TESTMO_RUN_NAME="${{ github.run_id }}"
+          EXTRA_TAG=""
+          ;;
+      esac
+
+      testmo automation:resources:add-link --name build --url $RUN_URL --resources testmo.json
+      testmo automation:resources:add-field --name git-sha --type string --value ${GITHUB_SHA:0:7} --resources testmo.json
+      testmo automation:run:create --instance "$TESTMO_URL" --project-id ${{inputs.testman_project_id}} --name "$TESTMO_RUN_NAME" \
+        --source "${{inputs.log_suffix}}" --resources testmo.json \
+        --tags "$BRANCH_TAG" --tags "$EXTRA_TAG"  | \
+        echo "runid=$(cat)" >> $GITHUB_OUTPUT
+
+  - name: Test
+    shell: bash
+    run: |
+      cd $WORKDIR/../build/ydb
+
+      echo "Stdout log (gzip archive): ${{inputs.aws_endpoint}}/${{inputs.aws_bucket}}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{steps.init.outputs.logfilename}}" >> $GITHUB_STEP_SUMMARY
+
+      # Sed removes coloring from the output
+      
+      TMPDIR=$WORKDIR/tmp GTEST_OUTPUT="xml:$TESTREPDIR/unittests/" Y_UNITTEST_OUTPUT="xml:$TESTREPDIR/unittests/" \
+        ctest -j28 --timeout 1200 --force-new-ctest-process --output-on-failure \
+              --output-junit $TESTREPDIR/suites/ctest_report.xml \
+              -L '${{inputs.test_label_regexp}}' | \
+        sed -e 's/\x1b\[[0-9;]*m//g' | \
+        tee >(gzip --stdout > $WORKDIR/artifacts/${{steps.init.outputs.logfilename}}) | \
+        grep -E '(Test\s*#.*\*\*\*|\[FAIL\])|.*tests passed,.*tests failed out of' | \
+        tee $WORKDIR/short.log
+  - name: Test history upload results
+    if: always() && inputs.testman_token
+    shell: bash
+    run: |
+      testmo automation:run:submit-thread \
+        --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} \
+        --results $TESTREPDIR/unittests/*.xml 
+      testmo automation:run:submit-thread \
+        --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}} \
+        --results $TESTREPDIR/suites/*.xml \
+        -- cat $WORKDIR/short.log
+      testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{steps.th.outputs.runid}}
+  - name: Upload S3
+    uses: shallwefootball/s3-upload-action@master
+    if: always()
+    with:
+      aws_key_id: ${{inputs.AWS_KEY_ID }}
+      aws_secret_access_key: ${{inputs.AWS_KEY_VALUE}}
+      aws_bucket: ${{inputs.aws_bucket}}
+      source_dir: artifacts
+      destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}'
+      endpoint: ${{inputs.aws_endpoint}}
diff --git a/.github/actions/test_python/action.yml b/.github/actions/test_python/action.yml
new file mode 100644
index 0000000000..f195e408f2
--- /dev/null
+++ b/.github/actions/test_python/action.yml
@@ -0,0 +1,62 @@
+name: test-python
+description: Run functional Python tests
+inputs:
+  log_suffix:
+    required: true
+    type: string
+  test_label_regexp:
+    required: false
+    type: string
+  aws_key_id:
+    required: true
+    type: string
+  aws_key_value: 
+    required: true
+    type: string
+  aws_bucket:
+    required: true
+    type: string
+  aws_endpoint:
+    required: true
+    type: string
+    
+runs:
+  using: "composite"
+  steps:
+  - name: Test
+    shell: bash
+    run: |
+      export source_root=$(pwd)
+      export build_root=$(pwd)/../build/
+      mkdir -p ../artifacts
+      rm -rf ../artifacts/*
+
+      echo "Stdout log (gzip archive): https://storage.yandexcloud.net/${{inputs.aws_bucket}}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.log_suffix}}-${{inputs.sanitizer}}-pytest-stdout.gz" >> $GITHUB_STEP_SUMMARY
+      cd ${source_root}/ydb/tests/functional/
+      bad_suites=$(grep -Eo 'ignore=[a-zA-Z_-]*' pytest.ini | sed -e 's/ignore=//g')
+      suites=""
+      for suite in $(echo */ | sed -e 's/\///g'); do
+        if [[ $(echo "$bad_suites" | grep -F -e $suite -) == '' ]]; then
+            suites+=$suite
+            suites+=$'\n'
+        fi
+      done
+      if [[ "${{inputs.test_label_regexp}}" != '' ]]; then
+        suites="${{inputs.test_label_regexp}}"
+      fi
+      source ${source_root}/ydb/tests/oss/launch/prepare.sh
+      echo -n "$suites" | parallel -j32 "pytest -o junit_logging=log -o junit_log_passing_tests=False \
+        -v --junit-xml=${source_root}/ydb/tests/functional/test-results/xml/{}/res.xml {}" | \
+        sed -e 's/\x1b\[[0-9;]*m//g' | \
+        tee >(gzip --stdout > ${source_root}/../artifacts/${{inputs.log_suffix}}-pytest-stdout.gz) | \
+        tee -a $GITHUB_STEP_SUMMARY
+  - name: Upload S3
+    uses: shallwefootball/s3-upload-action@master
+    if: always()
+    with:
+      aws_key_id: ${{inputs.aws_key_id}}
+      aws_secret_access_key: ${{inputs.aws_key_value}}
+      aws_bucket: ${{inputs.aws_bucket}}
+      source_dir: ../artifacts
+      destination_dir: '${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}'
+      endpoint: ${{inputs.aws_endpoint}}