diff options
| author | Kirill Rysin <35688753+naspirato@users.noreply.github.com> | 2024-12-23 18:48:18 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-12-23 20:48:18 +0300 |
| commit | d0bcedef0b7a33bffd9cce432ca59114fcca3dd6 (patch) | |
| tree | d33adf811348b19e8e478e07ec8ee6047eb310ee | |
| parent | 110099db410b29e5d0ae42964e6284713f7753a0 (diff) | |
| download | ydb-d0bcedef0b7a33bffd9cce432ca59114fcca3dd6.tar.gz | |
Batching for upload test results (#12838)
| -rwxr-xr-x | .github/scripts/analytics/upload_tests_results.py | 20 |
1 files changed, 12 insertions, 8 deletions
diff --git a/.github/scripts/analytics/upload_tests_results.py b/.github/scripts/analytics/upload_tests_results.py index b9a60f0577d..2c0fdfbadd3 100755 --- a/.github/scripts/analytics/upload_tests_results.py +++ b/.github/scripts/analytics/upload_tests_results.py @@ -218,9 +218,9 @@ def main(): test_results_file, build_type, job_name, job_id, commit, branch, pull, run_timestamp ) result_with_owners = get_codeowners_for_tests(codeowners, results) - prepared_for_update_rows = [] + prepared_for_upload_rows = [] for index, row in enumerate(result_with_owners): - prepared_for_update_rows.append({ + prepared_for_upload_rows.append({ 'branch': row['branch'], 'build_type': row['build_type'], 'commit': row['commit'], @@ -240,15 +240,19 @@ def main(): 'test_id': f"{row['pull']}_{row['run_timestamp']}_{index}", 'test_name': row['test_name'], }) - print(f'upserting runs: {len(prepared_for_update_rows)} rows') - if prepared_for_update_rows: + print(f'upserting runs: {len(prepared_for_upload_rows)} rows') + if prepared_for_upload_rows: + batch_rows_for_upload_size = 1000 with ydb.SessionPool(driver) as pool: create_tables(pool, test_table_name) - bulk_upsert(driver.table_client, full_path, - prepared_for_update_rows) - print('tests updated') + for start in range(0, len(prepared_for_upload_rows), batch_rows_for_upload_size): + batch_rows_for_upload = prepared_for_upload_rows[start:start + batch_rows_for_upload_size] + bulk_upsert(driver.table_client, full_path, + batch_rows_for_upload) + + print('tests uploaded') else: - print('nothing to upsert') + print('nothing to upload') |