diff options
Diffstat (limited to '.github/scripts/analytics')
4 files changed, 49 insertions, 24 deletions
diff --git a/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql b/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql index a56b3574cac..0aa77deae3a 100644 --- a/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql +++ b/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql @@ -132,12 +132,12 @@ SELECT WHEN Db LIKE '%static-node-1.ydb-cluster.com/Root/db%' THEN 'ansible_' WHEN Db LIKE '%ydb-vla-dev04-002%' THEN 'oltp-vla-perf1_' WHEN Db LIKE '%ydb-vla-dev04-005%' THEN 'oltp-vla-perf2_' - WHEN Db LIKE '%ydb-qa-01-klg-010%' THEN 'oltp-klg-perf3_' + WHEN Db LIKE '%ydb-qa-01-klg-015%' THEN 'oltp-klg-perf3_' WHEN Db LIKE '%ydb-qa-01-klg-014%' THEN 'oltp-klg-perf4_' - WHEN Db LIKE '%ydb-qa-01-klg-018%' THEN 'oltp-klg-perf5_' + WHEN Db LIKE '%ydb-qa-01-klg-022%' THEN 'oltp-klg-perf5_' WHEN Db LIKE '%ydb-qa-01-sas-000%' THEN 'oltp-3dc-perf6_' - WHEN Db LIKE '%ydb-qa-01-klg-021%' THEN 'oltp-klg-perf7_' - WHEN Db LIKE '%ydb-qa-01-klg-030%' THEN 'oltp-klg-perf9_' + WHEN Db LIKE '%ydb-qa-01-klg-035%' THEN 'oltp-klg-perf7_' + WHEN Db LIKE '%ydb-qa-01-klg-021%' THEN 'oltp-klg-perf9_' WHEN Db LIKE '%sas%' THEN 'sas_' WHEN Db LIKE '%vla%' THEN 'vla_' WHEN Db LIKE '%klg%' THEN 'klg_' diff --git a/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql b/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql index 9eb51df5577..a09a80d0ee1 100644 --- a/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql +++ b/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql @@ -141,12 +141,12 @@ SELECT WHEN s.Db LIKE '%static-node-1.ydb-cluster.com/Root/db%' THEN 'ansible_' WHEN s.Db LIKE '%ydb-vla-dev04-002%' THEN 'oltp-vla-perf1_' WHEN s.Db LIKE '%ydb-vla-dev04-005%' THEN 'oltp-vla-perf2_' - WHEN s.Db LIKE '%ydb-qa-01-klg-010%' THEN 'oltp-klg-perf3_' + WHEN s.Db LIKE '%ydb-qa-01-klg-015%' THEN 'oltp-klg-perf3_' WHEN s.Db LIKE '%ydb-qa-01-klg-014%' THEN 'oltp-klg-perf4_' - WHEN s.Db LIKE '%ydb-qa-01-klg-018%' THEN 'oltp-klg-perf5_' + WHEN s.Db LIKE '%ydb-qa-01-klg-022%' THEN 'oltp-klg-perf5_' WHEN s.Db LIKE '%ydb-qa-01-sas-000%' THEN 'oltp-3dc-perf6_' - WHEN s.Db LIKE '%ydb-qa-01-klg-021%' THEN 'oltp-klg-perf7_' - WHEN s.Db LIKE '%ydb-qa-01-klg-030%' THEN 'oltp-klg-perf9_' + WHEN s.Db LIKE '%ydb-qa-01-klg-035%' THEN 'oltp-klg-perf7_' + WHEN s.Db LIKE '%ydb-qa-01-klg-021%' THEN 'oltp-klg-perf9_' WHEN s.Db LIKE '%sas%' THEN 'sas_' WHEN s.Db LIKE '%vla%' THEN 'vla_' WHEN s.Db LIKE '%klg%' THEN 'klg_' diff --git a/.github/scripts/analytics/flaky_tests_history.py b/.github/scripts/analytics/flaky_tests_history.py index e3b3338059a..ce14e4e5bd9 100755 --- a/.github/scripts/analytics/flaky_tests_history.py +++ b/.github/scripts/analytics/flaky_tests_history.py @@ -21,7 +21,8 @@ def _dedupe_history_rows(rows): BASE_DATE = datetime.date(1970, 1, 1) -DEFAULT_MONTHS_BACK = 180 # 6 months +DEFAULT_DAYS_BACK = 30 # cold start when branch has no flaky_tests_window history +MAX_RESUME_GAP_DAYS = 180 # safety cap on incremental resume if collection was paused a long time def parse_arguments(): @@ -89,12 +90,17 @@ def determine_start_date(ydb_wrapper, test_runs_table, flaky_tests_table, build_ Logic: 1. If start_date_override is provided, use it (must be <= end_date) - 2. If history exists, use max_date_window from history (but not after end_date) - 3. If no history, check test_runs_table for min_run_date - 4. Default to 6 months ago from end_date + 2. If history exists, continue from max(date_window) (incremental; no cold-start cap), + but never look back further than MAX_RESUME_GAP_DAYS even if collection was + paused for a long time — avoids re-creating the unbounded-backfill problem this + cold-start rework was meant to fix, at the cost of leaving an un-backfilled gap + in the rare case collection was dormant that long. + 3. If no history, check test_runs_table for min_run_date, capped at DEFAULT_DAYS_BACK + 4. Default cold start to DEFAULT_DAYS_BACK ago from end_date """ end_date = end_date_override if end_date_override else datetime.date.today() - default_start_date = end_date - datetime.timedelta(days=DEFAULT_MONTHS_BACK) + default_start_date = end_date - datetime.timedelta(days=DEFAULT_DAYS_BACK) + max_resume_start_date = end_date - datetime.timedelta(days=MAX_RESUME_GAP_DAYS) # If user explicitly provided start_date_override, use it if start_date_override: @@ -106,13 +112,16 @@ def determine_start_date(ydb_wrapper, test_runs_table, flaky_tests_table, build_ max_date_window = get_max_date_from_history(ydb_wrapper, flaky_tests_table, build_type, branch) if max_date_window is not None: - # Clamp max_date_window to not exceed end_date before comparison - max_date_clamped = min(max_date_window, end_date) - # Use max_date_clamped if it's greater than default_start_date - if max_date_clamped > default_start_date: - start_date = max_date_clamped - else: - start_date = default_start_date + # Incremental resume: continue from the last stored day (do not apply cold-start + # lookback), capped so a long-dormant branch cannot trigger an unbounded backfill. + start_date = min(max_date_window, end_date) + if start_date < max_resume_start_date: + print( + f'⚠️ Last recorded date_window ({start_date}) is more than {MAX_RESUME_GAP_DAYS} days ' + f'old — history collection looks paused. Capping resume start to ' + f'{max_resume_start_date} instead of backfilling the full gap.' + ) + start_date = max_resume_start_date return start_date, end_date # No history exists, check test_runs_table for min date diff --git a/.github/scripts/analytics/tests_monitor.py b/.github/scripts/analytics/tests_monitor.py index 151ec444cff..de424b48a96 100755 --- a/.github/scripts/analytics/tests_monitor.py +++ b/.github/scripts/analytics/tests_monitor.py @@ -18,6 +18,8 @@ from github_issue_utils import ( ) from testowners_utils import normalize_github_team_owners_string +NEW_BRANCH_MONITOR_LOOKBACK_DAYS = 30 + def _dedupe_monitor_rows(rows): """One row per (full_name, date_window, branch, build_type); prefer deepest suite_folder.""" @@ -527,11 +529,25 @@ def main(): branch_creation_date = None if branch_creation_date: - process_start_date = max(branch_creation_date, default_start_date) - print(f"Found branch creation date: {branch_creation_date}") + cold_start_date = max( + today - datetime.timedelta(days=NEW_BRANCH_MONITOR_LOOKBACK_DAYS), + default_start_date, + ) + process_start_date = max(branch_creation_date, cold_start_date) + print( + f"Found branch creation date: {branch_creation_date}, " + f"collecting from {process_start_date} " + f"(lookback capped at {NEW_BRANCH_MONITOR_LOOKBACK_DAYS} days)" + ) else: - process_start_date = max(today - datetime.timedelta(days=7), default_start_date) - print(f"No test runs found for branch, using 1 week ago: {process_start_date}") + process_start_date = max( + today - datetime.timedelta(days=NEW_BRANCH_MONITOR_LOOKBACK_DAYS), + default_start_date, + ) + print( + f"No test runs found for branch, collecting from {process_start_date} " + f"({NEW_BRANCH_MONITOR_LOOKBACK_DAYS} days ago)" + ) date_list = [process_start_date + datetime.timedelta(days=x) for x in range((today - process_start_date).days + 1)] print(f"Init new monitor collecting from date {process_start_date}") |