diff options
| author | Bulat Gayazov <brgayazov@yandex-team.ru> | 2023-05-30 15:43:00 +0000 |
|---|---|---|
| committer | brgayazov <bulat@ydb.tech> | 2023-05-30 18:43:00 +0300 |
| commit | c822a501d30648062aa5806470924255cdd4d5a6 (patch) | |
| tree | b9dcc7aae463672c3a62face83a5a831b8779f4e /.github | |
| parent | aaba39f70f793486278aa4453d51119d3c19d256 (diff) | |
| download | ydb-c822a501d30648062aa5806470924255cdd4d5a6.tar.gz | |
Added parallelism to python tests CI
Added parallelism to python tests CI
Pull Request resolved: #230
Diffstat (limited to '.github')
| -rw-r--r-- | .github/actions/test_python/action.yml | 27 | ||||
| -rw-r--r-- | .github/workflows/build_and_test_ondemand.yml | 2 | ||||
| -rw-r--r-- | .github/workflows/prepare_vm_for_build.yml | 2 |
3 files changed, 19 insertions, 12 deletions
diff --git a/.github/actions/test_python/action.yml b/.github/actions/test_python/action.yml index aaee7e891b..f195e408f2 100644 --- a/.github/actions/test_python/action.yml +++ b/.github/actions/test_python/action.yml @@ -30,19 +30,26 @@ runs: export build_root=$(pwd)/../build/ mkdir -p ../artifacts rm -rf ../artifacts/* - source ${source_root}/ydb/tests/oss/launch/prepare.sh - echo "Stdout log (gzip archive): https://storage.yandexcloud.net/${{inputs.aws_bucket}}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.log_suffix}}-${{inputs.sanitizer}}-stdout.gz" >> $GITHUB_STEP_SUMMARY + echo "Stdout log (gzip archive): https://storage.yandexcloud.net/${{inputs.aws_bucket}}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.log_suffix}}-${{inputs.sanitizer}}-pytest-stdout.gz" >> $GITHUB_STEP_SUMMARY cd ${source_root}/ydb/tests/functional/ - export suite='' - if [[ "${{inputs.test_label_regexp}}" != "" ]]; then - suite="--suite ${{inputs.test_label_regexp}}" + bad_suites=$(grep -Eo 'ignore=[a-zA-Z_-]*' pytest.ini | sed -e 's/ignore=//g') + suites="" + for suite in $(echo */ | sed -e 's/\///g'); do + if [[ $(echo "$bad_suites" | grep -F -e $suite -) == '' ]]; then + suites+=$suite + suites+=$'\n' + fi + done + if [[ "${{inputs.test_label_regexp}}" != '' ]]; then + suites="${{inputs.test_label_regexp}}" fi - python ${source_root}/ydb/tests/oss/launch/launch.py --test-dir ${source_root}/ydb/tests/functional \ - --xml-dir ${source_root}/ydb/tests/functional/test-results/xml $suite | \ - sed -e 's/\x1b\[[0-9;]*m//g' | \ - tee >(gzip --stdout > ${source_root}/../artifacts/${{inputs.log_suffix}}-pytest-stdout.gz) | \ - tee -a $GITHUB_STEP_SUMMARY + source ${source_root}/ydb/tests/oss/launch/prepare.sh + echo -n "$suites" | parallel -j32 "pytest -o junit_logging=log -o junit_log_passing_tests=False \ + -v --junit-xml=${source_root}/ydb/tests/functional/test-results/xml/{}/res.xml {}" | \ + sed -e 's/\x1b\[[0-9;]*m//g' | \ + tee >(gzip --stdout > ${source_root}/../artifacts/${{inputs.log_suffix}}-pytest-stdout.gz) | \ + tee -a $GITHUB_STEP_SUMMARY - name: Upload S3 uses: shallwefootball/s3-upload-action@master if: always() diff --git a/.github/workflows/build_and_test_ondemand.yml b/.github/workflows/build_and_test_ondemand.yml index 022fa593c8..bc3a01ff34 100644 --- a/.github/workflows/build_and_test_ondemand.yml +++ b/.github/workflows/build_and_test_ondemand.yml @@ -111,7 +111,7 @@ jobs: if: inputs.run_functional_tests && (success() || failure()) && steps.build.conclusion != 'failure' uses: ./.github/actions/test_python with: - log_suffix: ${{format('{0}-{1}', 'X64', inputs.sanitizer)}} + log_suffix: ${{format('{0}{1}', 'X64', inputs.sanitizer)}} test_label_regexp: ${{ inputs.test_label_regexp }} aws_key_id: ${{secrets.AWS_KEY_ID}} aws_key_value: ${{secrets.AWS_KEY_VALUE}} diff --git a/.github/workflows/prepare_vm_for_build.yml b/.github/workflows/prepare_vm_for_build.yml index 0cee339c69..dab4270299 100644 --- a/.github/workflows/prepare_vm_for_build.yml +++ b/.github/workflows/prepare_vm_for_build.yml @@ -27,7 +27,7 @@ jobs: echo "deb http://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null sudo apt-get update - sudo apt-get -y install python-is-python3 git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev + sudo apt-get -y install python-is-python3 git cmake python3-pip ninja-build antlr3 m4 clang-12 lld-12 libidn11-dev libaio1 libaio-dev parallel sudo pip3 install conan==1.59 pytest==7.1.3 grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow - name: Install AllureCtl if: false |