1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
name: Create issues for muted tests
on:
pull_request:
types:
- closed
branches:
- main
workflow_dispatch:
inputs:
pr_number:
description: 'The pull request number (for comment / append body)'
required: true
type: number
base_branch:
description: 'Branch for YDB scripts and create_issues --branch (PR runs use the merged PR base ref)'
required: false
type: string
default: main
build_type:
description: 'Optional single build type override (empty = use build types from mute_issue_and_digest_config.json)'
required: false
type: string
default: ''
env:
GH_TOKEN: ${{ secrets.YDBOT_TOKEN }}
WORKFLOW_CHECKOUT_REF: ${{ github.ref_name }}
jobs:
resolve-build-types:
runs-on: [ self-hosted, auto-provisioned, build-preset-analytic-node]
outputs:
base_branch: ${{ steps.resolve.outputs.base_branch }}
matrix_build_types: ${{ steps.resolve.outputs.matrix_build_types }}
steps:
- name: Checkout config
uses: actions/checkout@v5
with:
ref: ${{ env.WORKFLOW_CHECKOUT_REF }}
- name: Resolve build types matrix
id: resolve
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
BASE_BRANCH="${{ github.event.inputs.base_branch }}"
else
BASE_BRANCH="${{ github.event.pull_request.base.ref || github.base_ref }}"
if [ -z "$BASE_BRANCH" ]; then
echo "BASE_BRANCH is empty for pull_request event" >&2
exit 1
fi
fi
echo "base_branch=$BASE_BRANCH" >> "$GITHUB_OUTPUT"
if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.build_type }}" ]; then
MATRIX_BUILD_TYPES='["${{ github.event.inputs.build_type }}"]'
else
MATRIX_BUILD_TYPES="$(python3 .github/scripts/tests/mute/mute_helper.py issue-build-types \
--profiles-config ".github/config/mute_issue_and_digest_config.json" \
--branch "$BASE_BRANCH")"
fi
echo "matrix_build_types=$MATRIX_BUILD_TYPES" >> "$GITHUB_OUTPUT"
create-issues-for-muted-tests:
needs: resolve-build-types
runs-on: [ self-hosted, auto-provisioned, build-preset-analytic-node]
if: |
((github.event_name == 'pull_request' &&
contains(github.event.pull_request.labels.*.name, 'mute-unmute')) ||
github.event_name == 'workflow_dispatch')
strategy:
fail-fast: false
matrix:
build_type: ${{ fromJson(needs.resolve-build-types.outputs.matrix_build_types) }}
steps:
- name: Set environment variables for branches
run: |
echo "BASE_BRANCH=${{ needs.resolve-build-types.outputs.base_branch }}" >> "$GITHUB_ENV"
- name: Checkout repository
uses: actions/checkout@v5
with:
ref: ${{ env.WORKFLOW_CHECKOUT_REF }}
- name: Resolve and load muted YA from base branch
run: |
set -euo pipefail
MUTED_YA_RELATIVE="$(python3 .github/scripts/tests/mute/mute_helper.py resolve-path --preset "${{ matrix.build_type }}")"
git fetch origin "${{ env.BASE_BRANCH }}:refs/remotes/origin/${{ env.BASE_BRANCH }}"
git show "origin/${{ env.BASE_BRANCH }}:${MUTED_YA_RELATIVE}" > base_muted_ya.txt
echo "MUTED_YA_FILE_PATH=${{ github.workspace }}/base_muted_ya.txt" >> "$GITHUB_ENV"
echo "Using base_branch=${{ env.BASE_BRANCH }} build_type=${{ matrix.build_type }} muted_ya_file=$MUTED_YA_RELATIVE"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ydb[yc] PyGithub requests matplotlib numpy pandas
- name: Setup ydb access
uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
with:
ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
ydb_qa_config: ${{ vars.YDB_QA_CONFIG }}
- name: Create issues for muted tests
id: create_issues
env:
GITHUB_TOKEN: ${{ env.GH_TOKEN }}
ENABLE_NEED_AI_REVIEW_LABEL: ${{ vars.ENABLE_NEED_AI_REVIEW_LABEL || '' }}
run: .github/scripts/tests/mute/create_new_muted_ya.py create_issues --file_path="${{ env.MUTED_YA_FILE_PATH }}" --branch=${{ env.BASE_BRANCH }} --build-type=${{ matrix.build_type }}
- name: Add issues to PR
continue-on-error: true
env:
GITHUB_TOKEN: ${{ env.GH_TOKEN }}
run: python .github/scripts/create_or_update_pr.py append_pr_body --pr_number=${{ github.event.pull_request.number || github.event.inputs.pr_number }} --body=${{ steps.create_issues.outputs.created_issues_file }}
- name: Comment PR
uses: actions/github-script@v8
with:
github-token: ${{ env.GH_TOKEN }}
script: |
const fs = require('fs');
const path = require('path');
const workflowUrl = `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`;
const filePath = '${{ steps.create_issues.outputs.created_issues_file }}';
const bodyText = fs.readFileSync(filePath, 'utf8');
const completeBody = `Collected in workflow [#${{ github.run_number }}](${workflowUrl})\n\n${bodyText}`;
github.rest.issues.createComment({
issue_number: ${{ github.event.pull_request.number || github.event.inputs.pr_number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: completeBody
});
- name: Update muted tests in DB
run: python3 .github/scripts/tests/mute/get_muted_tests.py upload_muted_tests --branch=${{ env.BASE_BRANCH }} --build_type=${{ matrix.build_type }} --muted_ya_file="${{ env.MUTED_YA_FILE_PATH }}"
- name: Collect test history data
run: python3 .github/scripts/analytics/flaky_tests_history.py --branch=${{ env.BASE_BRANCH }} --build_type=${{ matrix.build_type }}
- name: Update test monitor
run: python3 .github/scripts/analytics/tests_monitor.py --branch=${{ env.BASE_BRANCH }} --build_type=${{ matrix.build_type }}
# Issue export, mapping, and muted mart are handled by collect_analytics_fast (every 30 min)
sync-manual-fast-unmute:
needs: [resolve-build-types, create-issues-for-muted-tests]
runs-on: [ self-hosted, auto-provisioned, build-preset-analytic-node]
if: |
((github.event_name == 'pull_request' &&
contains(github.event.pull_request.labels.*.name, 'mute-unmute')) ||
github.event_name == 'workflow_dispatch')
steps:
- name: Set environment variables for branches
run: |
echo "BASE_BRANCH=${{ needs.resolve-build-types.outputs.base_branch }}" >> "$GITHUB_ENV"
- name: Checkout repository
uses: actions/checkout@v5
with:
ref: ${{ env.WORKFLOW_CHECKOUT_REF }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ydb[yc] requests
- name: Setup ydb access
uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
with:
ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
ydb_qa_config: ${{ vars.YDB_QA_CONFIG }}
- name: Export GitHub issues to YDB
env:
GITHUB_TOKEN: ${{ env.GH_TOKEN }}
run: python3 .github/scripts/analytics/export_issues_to_ydb.py
- name: Sync manual fast-unmute state
env:
GITHUB_TOKEN: ${{ env.GH_TOKEN }}
run: python3 .github/scripts/tests/mute/manual_unmute.py sync
|
