diff options
author | Kirill Rysin <35688753+naspirato@users.noreply.github.com> | 2024-09-24 17:15:31 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-09-24 18:15:31 +0300 |
commit | 332843a04ebba4d6b9672cb296dcc81795f4519c (patch) | |
tree | 237eec71a4db145edeaa01e15b8cdb754ab47095 | |
parent | 18174b04e8714e997619817309f894b524832178 (diff) | |
download | ydb-332843a04ebba4d6b9672cb296dcc81795f4519c.tar.gz |
Mute tools + mute info in for pr (#9714)
-rw-r--r-- | .github/actions/test_ya/action.yml | 42 | ||||
-rw-r--r-- | .github/scripts/analytics/get_mute_issues.py | 226 | ||||
-rw-r--r-- | .github/scripts/tests/get_diff_lines_of_file.py | 52 | ||||
-rw-r--r-- | .github/scripts/tests/get_muted_tests.py | 295 | ||||
-rwxr-xr-x | .github/scripts/tests/transform_ya_junit.py (renamed from .github/scripts/tests/transform-ya-junit.py) | 0 |
5 files changed, 613 insertions, 2 deletions
diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index bc9b9a3642..5a1429237a 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -399,7 +399,7 @@ runs: gzip -c $CURRENT_JUNIT_XML_PATH > $CURRENT_PUBLIC_DIR/orig_junit.xml.gz # postprocess junit report - .github/scripts/tests/transform-ya-junit.py -i \ + .github/scripts/tests/transform_ya_junit.py -i \ -m .github/config/muted_ya.txt \ --ya_out "$YA_MAKE_OUT_DIR" \ --public_dir "$PUBLIC_DIR" \ @@ -533,6 +533,44 @@ runs: run: | .github/scripts/tests/fail-checker.py "$LAST_JUNIT_REPORT_XML" + - name: show diff mute_ya.txt + if: inputs.build_preset == 'relwithdebinfo' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') + shell: bash + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ORIGINAL_HEAD=$(git rev-parse HEAD) + get_file_diff_script=.github/scripts/tests/get_diff_lines_of_file.py + file_to_check=.github/config/muted_ya.txt + check_result=`$get_file_diff_script --base_sha $ORIGINAL_HEAD~1 --head_sha $ORIGINAL_HEAD --file_path $file_to_check` + if [[ ${check_result} == *"not changed" ]];then + echo file ${file_to_check} NOT changed + else + echo file ${file_to_check} changed + .github/scripts/tests/get_muted_tests.py --output_folder "$PUBLIC_DIR/mute_info/" get_mute_diff --base_sha $ORIGINAL_HEAD~1 --head_sha $ORIGINAL_HEAD --job-id "${{ github.run_id }}" --branch "${GITHUB_REF_NAME}" + FILE_PATH=$PUBLIC_DIR/mute_info/2_new_muted_tests.txt + SEPARATOR="" + if [ -f "$FILE_PATH" ]; then + LINE_COUNT=$(wc -l < "$FILE_PATH") + if [ "$LINE_COUNT" -gt 0 ]; then + SEPARATOR=', ' + MESSAGE="Muted new $LINE_COUNT [tests](${PUBLIC_DIR_URL}/mute_info/2_new_muted_tests.txt)" + fi + fi + FILE_PATH=$PUBLIC_DIR/mute_info/3_unmuted_tests.txt + if [ -f "$FILE_PATH" ]; then + LINE_COUNT_unmute=$(wc -l < "$FILE_PATH") + if [ "$LINE_COUNT_unmute" -gt 0 ]; then + MESSAGE="${MESSAGE}${SEPARATOR}Unmuted $LINE_COUNT_unmute [tests](${PUBLIC_DIR_URL}/mute_info/3_unmuted_tests.txt)" + fi + fi + if [ -n "$MESSAGE" ]; then + printf "$MESSAGE" | .github/scripts/tests/comment-pr.py --color orange + fi + fi + + - name: sync results to s3 and publish links if: always() shell: bash @@ -602,4 +640,4 @@ runs: env: BUILD_PRESET: ${{ inputs.build_preset }} GITHUB_TOKEN: ${{ github.token }} - run: echo "Check cancelled" | .github/scripts/tests/comment-pr.py --color black + run: echo "Check cancelled" | .github/scripts/tests/comment-pr.py --color black
\ No newline at end of file diff --git a/.github/scripts/analytics/get_mute_issues.py b/.github/scripts/analytics/get_mute_issues.py new file mode 100644 index 0000000000..e2797f0658 --- /dev/null +++ b/.github/scripts/analytics/get_mute_issues.py @@ -0,0 +1,226 @@ +import os +import re +import requests + +ORG_NAME = 'ydb-platform' +PROJECT_ID = '45' +query_template = """ +{ + organization(login: "%s") { + projectV2(number: %s) { + id + title + items(first: 100, after: %s) { + nodes { + content { + ... on Issue { + id + title + url + state + body + createdAt + } + } + fieldValues(first: 20) { + nodes { + ... on ProjectV2ItemFieldSingleSelectValue { + field { + ... on ProjectV2SingleSelectField { + name + } + } + name + id + updatedAt + } + ... on ProjectV2ItemFieldLabelValue { + labels(first: 20) { + nodes { + id + name + } + } + } + ... on ProjectV2ItemFieldTextValue { + text + id + updatedAt + creator { + url + } + } + ... on ProjectV2ItemFieldMilestoneValue { + milestone { + id + } + } + ... on ProjectV2ItemFieldRepositoryValue { + repository { + id + url + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } +} +""" + + +def run_query(query, headers): + request = requests.post('https://api.github.com/graphql', json={'query': query}, headers=headers) + if request.status_code == 200: + return request.json() + else: + raise Exception(f"Query failed to run by returning code of {request.status_code}. {query}") + + +def fetch_all_issues(org_name, project_id): + issues = [] + has_next_page = True + end_cursor = "null" + + while has_next_page: + query = query_template % (org_name, project_id, end_cursor) + GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] + headers = {"Authorization": f"Bearer {GITHUB_TOKEN}"} + result = run_query(query, headers) + + if result: + project_items = result['data']['organization']['projectV2']['items'] + issues.extend(project_items['nodes']) + + page_info = project_items['pageInfo'] + has_next_page = page_info['hasNextPage'] + end_cursor = f"\"{page_info['endCursor']}\"" if page_info['endCursor'] else "null" + else: + has_next_page = False + + return issues + + +def parse_body(body): + tests = [] + branches = [] + prepared_body = '' + start_mute_list = "<!--mute_list_start-->" + end_mute_list = "<!--mute_list_end-->" + start_branch_list = "<!--branch_list_start-->" + end_branch_list = "<!--branch_list_end-->" + + # tests + if all(x in body for x in [start_mute_list, end_mute_list]): + idx1 = body.find(start_mute_list) + idx2 = body.find(end_mute_list) + lines = body[idx1 + len(start_mute_list) + 1 : idx2].split('\n') + else: + if body.startswith('Mute:'): + prepared_body = body.split('Mute:', 1)[1].strip() + elif body.startswith('Mute'): + prepared_body = body.split('Mute', 1)[1].strip() + elif body.startswith('ydb'): + prepared_body = body + lines = prepared_body.split('**Add line to')[0].split('\n') + tests = [line.strip() for line in lines if line.strip().startswith('ydb/')] + + # branch + if all(x in body for x in [start_branch_list, end_branch_list]): + idx1 = body.find(start_branch_list) + idx2 = body.find(end_branch_list) + branches = body[idx1 + len(start_branch_list) + 1 : idx2].split('\n') + else: + branches = ['main'] + + return tests, branches + + +def get_issues_and_tests_from_project(ORG_NAME, PROJECT_ID): + issues = fetch_all_issues(ORG_NAME, PROJECT_ID) + issues_prepared = {} + for issue in issues: + content = issue['content'] + if content: + body = content['body'] + + # for debug + if content['id'] == 'I_kwDOGzZjoM6V3BoE': + print(1) + # + + tests, branches = parse_body(body) + + field_values = issue.get('fieldValues', {}).get('nodes', []) + for field_value in field_values: + field_name = field_value.get('field', {}).get('name', '').lower() + + if field_name == "status" and 'name' in field_value: + status = field_value.get('name', 'N/A') + status_updated = field_value.get('updatedAt', '1970-01-0901T00:00:01Z') + elif field_name == "owner" and 'name' in field_value: + owner = field_value.get('name', 'N/A') + + print(f"Issue ID: {content['id']}") + print(f"Title: {content['title']}") + print(f"URL: {content['url']}") + print(f"State: {content['state']}") + print(f"CreatedAt: {content['createdAt']}") + print(f"Status: {status}") + print(f"Status updated: {status_updated}") + print(f"Owner: {owner}") + print("Tests:") + + issues_prepared[content['id']] = {} + issues_prepared[content['id']]['title'] = content['title'] + issues_prepared[content['id']]['url'] = content['url'] + issues_prepared[content['id']]['state'] = content['state'] + issues_prepared[content['id']]['createdAt'] = content['createdAt'] + issues_prepared[content['id']]['status_updated'] = status_updated + issues_prepared[content['id']]['status'] = status + issues_prepared[content['id']]['owner'] = owner + issues_prepared[content['id']]['tests'] = [] + issues_prepared[content['id']]['branches'] = branches + + for test in tests: + issues_prepared[content['id']]['tests'].append(test) + print(f"- {test}") + print('\n') + + return issues_prepared + + +def get_muted_tests(): + issues = get_issues_and_tests_from_project(ORG_NAME, PROJECT_ID) + muted_tests = {} + for issue in issues: + if issues[issue]["status"] == "Muted": + for test in issues[issue]['tests']: + if test not in muted_tests: + muted_tests[test] = [] + muted_tests[test].append( + { + 'url': issues[issue]['url'], + 'createdAt': issues[issue]['createdAt'], + 'status_updated': issues[issue]['status_updated'], + } + ) + + return muted_tests + + +def main(): + if "GITHUB_TOKEN" not in os.environ: + print("Error: Env variable GITHUB_TOKEN is missing, skipping") + return 1 + get_muted_tests() + + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/.github/scripts/tests/get_diff_lines_of_file.py b/.github/scripts/tests/get_diff_lines_of_file.py new file mode 100644 index 0000000000..3925215aa0 --- /dev/null +++ b/.github/scripts/tests/get_diff_lines_of_file.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import requests +import subprocess +import sys + + +def get_diff_lines_of_file(base_sha, head_sha, file_path): + print(f"base_sha: {base_sha}") + print(f"head_sha: {head_sha}") + print(f"file_path: {file_path}") + + # Use git to get two versions of file + result_base = subprocess.run(['git', 'show', base_sha + ':' + file_path], capture_output=True, text=True) + if result_base.returncode != 0: + raise RuntimeError(f"Error running git show: {result_base.stderr}") + + result_head = subprocess.run(['git', 'show', head_sha + ':' + file_path], capture_output=True, text=True) + if result_head.returncode != 0: + raise RuntimeError(f"Error running git show: {result_base.stderr}") + + base_set_lines = set([line for line in result_base.stdout.splitlines() if line]) + head_set_lines = set([line for line in result_head.stdout.splitlines() if line]) + added_lines = list(head_set_lines - base_set_lines) + removed_lines = list(base_set_lines - head_set_lines) + print("\n### Added Lines:") + print("\n".join(added_lines)) + print("\n### Removed Lines:") + print("\n".join(removed_lines)) + return added_lines, removed_lines + + +def main(base_sha, head_sha, file_path): + added_lines, removed_lines = get_diff_lines_of_file(base_sha, head_sha, file_path) + if added_lines or removed_lines: + print(f"file {file_path} changed") + else: + print(f"file {file_path} not changed") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Returns added and removed lines for file compared by git diff in two commit sha's" + ) + parser.add_argument('--base_sha', type=str, required=True) + parser.add_argument('--head_sha', type=str, required=True) + parser.add_argument('--file_path', type=str, required=True) + args = parser.parse_args() + + main(args.base_sha, args.head_sha, args.file_path)
\ No newline at end of file diff --git a/.github/scripts/tests/get_muted_tests.py b/.github/scripts/tests/get_muted_tests.py new file mode 100644 index 0000000000..e6a76132c8 --- /dev/null +++ b/.github/scripts/tests/get_muted_tests.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +import argparse +import configparser +import datetime +import os +import posixpath +import re +import ydb +from get_diff_lines_of_file import get_diff_lines_of_file +from mute_utils import pattern_to_re +from transform_ya_junit import YaMuteCheck + +dir = os.path.dirname(__file__) +config = configparser.ConfigParser() +config_file_path = f"{dir}/../../config/ydb_qa_db.ini" +repo_path = f"{dir}/../../../" +muted_ya_path = '.github/config/muted_ya.txt' +config.read(config_file_path) + +DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"] +DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"] + + +def get_all_tests(job_id=None, branch=None): + print(f'Getting all tests') + + with ydb.Driver( + endpoint=DATABASE_ENDPOINT, + database=DATABASE_PATH, + credentials=ydb.credentials_from_env_variables(), + ) as driver: + driver.wait(timeout=10, fail_fast=True) + + # settings, paths, consts + tc_settings = ydb.TableClientSettings().with_native_date_in_result_sets(enabled=True) + table_client = ydb.TableClient(driver, tc_settings) + + # geting last date from history + today = datetime.date.today().strftime('%Y-%m-%d') + if job_id and branch: # extend all tests from main by new tests from pr + + tests = f""" + SELECT * FROM ( + SELECT + suite_folder, + test_name, + full_name + from `test_results/analytics/testowners` + WHERE + run_timestamp_last >= Date('{today}') - 6*Interval("P1D") + and run_timestamp_last <= Date('{today}') + Interval("P1D") + UNION + SELECT DISTINCT + suite_folder, + test_name, + suite_folder || '/' || test_name as full_name + FROM `test_results/test_runs_column` + WHERE + job_id = {job_id} + and branch = '{branch}' + ) + """ + else: # only all tests from main + tests = f""" + SELECT + suite_folder, + test_name, + full_name, + owners, + run_timestamp_last, + Date('{today}') as date + FROM `test_results/analytics/testowners` + WHERE + run_timestamp_last >= Date('{today}') - 6*Interval("P1D") + and run_timestamp_last <= Date('{today}') + Interval("P1D") + """ + query = ydb.ScanQuery(tests, {}) + it = table_client.scan_query(query) + results = [] + while True: + try: + result = next(it) + results = results + result.result_set.rows + except StopIteration: + break + return results + + +def create_tables(pool, table_path): + print(f"> create table if not exists:'{table_path}'") + + def callee(session): + session.execute_scheme( + f""" + CREATE table IF NOT EXISTS `{table_path}` ( + `date` Date NOT NULL, + `test_name` Utf8 NOT NULL, + `suite_folder` Utf8 NOT NULL, + `full_name` Utf8 NOT NULL, + `run_timestamp_last` Timestamp NOT NULL, + `owners` Utf8, + `branch` Utf8 NOT NULL, + `is_muted` Uint32 , + PRIMARY KEY (`date`,branch, `test_name`, `suite_folder`, `full_name`) + ) + PARTITION BY HASH(date,branch) + WITH (STORE = COLUMN) + """ + ) + + return pool.retry_operation_sync(callee) + + +def bulk_upsert(table_client, table_path, rows): + print(f"> bulk upsert: {table_path}") + column_types = ( + ydb.BulkUpsertColumns() + .add_column("date", ydb.OptionalType(ydb.PrimitiveType.Date)) + .add_column("test_name", ydb.OptionalType(ydb.PrimitiveType.Utf8)) + .add_column("suite_folder", ydb.OptionalType(ydb.PrimitiveType.Utf8)) + .add_column("full_name", ydb.OptionalType(ydb.PrimitiveType.Utf8)) + .add_column("run_timestamp_last", ydb.OptionalType(ydb.PrimitiveType.Timestamp)) + .add_column("owners", ydb.OptionalType(ydb.PrimitiveType.Utf8)) + .add_column("branch", ydb.OptionalType(ydb.PrimitiveType.Utf8)) + .add_column("is_muted", ydb.OptionalType(ydb.PrimitiveType.Uint32)) + ) + table_client.bulk_upsert(table_path, rows, column_types) + + +def write_to_file(text, file): + os.makedirs(os.path.dirname(file), exist_ok=True) + with open(file, 'w') as f: + f.writelines(text) + + +def upload_muted_tests(tests): + with ydb.Driver( + endpoint=DATABASE_ENDPOINT, + database=DATABASE_PATH, + credentials=ydb.credentials_from_env_variables(), + ) as driver: + driver.wait(timeout=10, fail_fast=True) + + # settings, paths, consts + tc_settings = ydb.TableClientSettings().with_native_date_in_result_sets(enabled=True) + table_client = ydb.TableClient(driver, tc_settings) + + table_path = f'test_results/all_tests_with_owner_and_mute' + + with ydb.SessionPool(driver) as pool: + create_tables(pool, table_path) + full_path = posixpath.join(DATABASE_PATH, table_path) + bulk_upsert(driver.table_client, full_path, tests) + + +def to_str(data): + if isinstance(data, str): + return data + elif isinstance(data, bytes): + return data.decode('utf-8') + else: + raise ValueError("Unsupported type") + + +def mute_applier(args): + output_path = args.output_folder + + all_tests_file = os.path.join(output_path, '1_all_tests.txt') + all_muted_tests_file = os.path.join(output_path, '1_all_muted_tests.txt') + + if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ: + print("Error: Env variable CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping") + return 1 + else: + # Do not set up 'real' variable from gh workflows because it interfere with ydb tests + # So, set up it locally + os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = os.environ[ + "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" + ] + + # all muted + mute_check = YaMuteCheck() + mute_check.load(muted_ya_path) + + if args.mode == 'upload_muted_tests': + all_tests = get_all_tests(branch=args.branch) + for test in all_tests: + testsuite = to_str(test['suite_folder']) + testcase = to_str(test['test_name']) + test['branch'] = 'main' + test['is_muted'] = int(mute_check(testsuite, testcase)) + + upload_muted_tests(all_tests) + + elif args.mode == 'get_mute_diff': + all_tests = get_all_tests(job_id=args.job_id, branch=args.branch) + all_tests.sort(key=lambda test: test['full_name']) + muted_tests = [] + all_tests_names_and_suites = [] + for test in all_tests: + testsuite = to_str(test['suite_folder']) + testcase = to_str(test['test_name']) + all_tests_names_and_suites.append(testsuite + ' ' + testcase + '\n') + if mute_check(testsuite, testcase): + muted_tests.append(testsuite + ' ' + testcase + '\n') + + write_to_file(all_tests_names_and_suites, all_tests_file) + write_to_file(muted_tests, all_muted_tests_file) + + added_mute_lines_file = os.path.join(output_path, '2_added_mute_lines.txt') + new_muted_tests_file = os.path.join(output_path, '2_new_muted_tests.txt') + removed_mute_lines_file = os.path.join(output_path, '3_removed_mute_lines.txt') + unmuted_tests_file = os.path.join(output_path, '3_unmuted_tests.txt') + + added_lines, removed_lines = get_diff_lines_of_file(args.base_sha, args.head_sha, muted_ya_path) + + # checking added lines + write_to_file('\n'.join(added_lines), added_mute_lines_file) + mute_check.load(added_mute_lines_file) + added_muted_tests = [] + print("New muted tests captured") + for test in all_tests: + testsuite = to_str(test['suite_folder']) + testcase = to_str(test['test_name']) + if mute_check(testsuite, testcase): + added_muted_tests.append(testsuite + ' ' + testcase + '\n') + + # checking removed lines + write_to_file('\n'.join(removed_lines), removed_mute_lines_file) + mute_check.load(removed_mute_lines_file) + removed_muted_tests = [] + print("Unmuted tests captured") + for test in all_tests: + testsuite = to_str(test['suite_folder']) + testcase = to_str(test['test_name']) + if mute_check(testsuite, testcase): + removed_muted_tests.append(testsuite + ' ' + testcase + '\n') + + # geting only uniq items in both lists because not uniq items= this tests was muted before + added_set = set(added_muted_tests) + removed_set = set(removed_muted_tests) + added_unique = added_set - removed_set + removed_unique = removed_set - added_set + added_muted_tests = list(sorted(added_unique)) + removed_muted_tests = list(sorted(removed_unique)) + + write_to_file(added_muted_tests, new_muted_tests_file) + write_to_file(removed_muted_tests, unmuted_tests_file) + + print(f"All tests have been written to {all_tests_file}.") + print(f"All mutes tests have been written to {all_muted_tests_file}.") + print(f"Added lines have been written to {added_mute_lines_file}.") + print(f"New muted tests have been written to {new_muted_tests_file}.") + print(f"Removed lines have been written to {removed_mute_lines_file}.") + print(f"Unmuted tests have been written to {unmuted_tests_file}.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate diff files for mute_ya.txt") + + parser.add_argument( + '--output_folder', + type=str, + default=repo_path + '.github/config/mute_info/', + help=f'The folder to output results. Default is the value of repo_path = {repo_path}.github/config/mute_info/.', + ) + + subparsers = parser.add_subparsers(dest='mode', help="Mode to perform") + + upload_muted_tests_parser = subparsers.add_parser( + 'upload_muted_tests', help='apply mute rules for all tests in main and upload to database' + ) + upload_muted_tests_parser.add_argument( + '--branch', required=True, default='main', help='branch for getting all tests' + ) + + get_mute_details_parser = subparsers.add_parser( + 'get_mute_diff', + help='apply mute rules for all tests in main extended by new tests from pr and collect new muted and unmuted', + ) + get_mute_details_parser.add_argument('--base_sha', required=True, help='Base sha of PR') + get_mute_details_parser.add_argument('--head_sha', required=True, help='Head sha of PR') + get_mute_details_parser.add_argument( + '--branch', + required=True, + help='pass branch to extend list of tests by new tests from this pr (by job-id of PR-check and branch)', + ) + get_mute_details_parser.add_argument( + '--job-id', + required=True, + help='pass job-id to extend list of tests by new tests from this pr (by job-id of PR-check and branch)', + ) + args = parser.parse_args() + + mute_applier(args)
\ No newline at end of file diff --git a/.github/scripts/tests/transform-ya-junit.py b/.github/scripts/tests/transform_ya_junit.py index 05d6927d38..05d6927d38 100755 --- a/.github/scripts/tests/transform-ya-junit.py +++ b/.github/scripts/tests/transform_ya_junit.py |