diff options
Diffstat (limited to '.github/scripts')
15 files changed, 1182 insertions, 165 deletions
diff --git a/.github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_full.sql b/.github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_full.sql index 9c2711e5c4e..7240606e214 100644 --- a/.github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_full.sql +++ b/.github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_full.sql @@ -1,114 +1,216 @@ --- For full reload via data_mart_executor_by_month.py. Issues on a daily timeline: for each date, which issues are open at end of day and which were closed that day. --- Run: python3 .github/scripts/analytics/data_mart_executor_by_month.py --query_path .github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_full.sql --table_path test_results/analytics/github_issues_timeline --store_type column --partition_keys date --primary_keys date issue_number project_item_id --- Optional: --by_month 12 (default) --- FULL WINDOW: use with data_mart_executor (full 365-day window) or data_mart_executor_by_month (per-month). --- Dates come from tests_monitor (date_window), no ListFromRange/FLATTEN BY. --- In BI: filter by date, owner_team; for issue list per day — filter by date; for counts — GROUP BY date, SUM(is_open_at_end_of_day), SUM(closed_on_this_day). --- --- Windows (change here or override via script): --- $timeline_days — date dimension and "open in window": include issues open on at least one day in [now - timeline_days, now]. --- Include: created_date <= today and (still open or closed within window: closed_at >= now - timeline_days). -$timeline_days = 365; --- For by_month wrapper: script overwrites these to restrict to one month (avoids connection timeouts) -$month_start = Date("1970-01-01"); -$month_end = Date("2100-01-01"); - -SELECT - dt.d AS date, - i.project_item_id AS project_item_id, - i.issue_id AS issue_id, - i.issue_number AS issue_number, - i.title AS title, - i.url AS url, - i.state AS state, - i.state_reason AS state_reason, - i.created_at AS created_at, - i.updated_at AS updated_at, - i.closed_at AS closed_at, - i.created_date AS created_date, - i.updated_date AS updated_date, - i.author_login AS author_login, - i.author_url AS author_url, - i.repository_name AS repository_name, - i.repository_url AS repository_url, - i.project_status AS project_status, - i.project_owner AS project_owner, - i.project_priority AS project_priority, - i.is_in_project AS is_in_project, - i.days_since_created AS days_since_created, - i.days_since_updated AS days_since_updated, - i.time_to_close_hours AS time_to_close_hours, - i.assignees AS assignees, - i.labels AS labels, - i.milestone AS milestone, - i.project_fields AS project_fields, - i.info AS info, - i.issue_type AS issue_type, - i.exported_at AS exported_at, - i.owner_team AS owner_team, - i.labels_list AS labels_list, - i.max_branch AS max_branch, - i.env AS env, - i.priority AS priority, - i.releaseblocker_state AS releaseblocker_state, - i.branch AS branch, - i.area AS area, - CAST( - (i.closed_at IS NULL OR Cast(i.closed_at AS Date) > dt.d) AS Uint8 - ) AS is_open_at_end_of_day, - CAST( - (i.closed_at IS NOT NULL AND Cast(i.closed_at AS Date) = dt.d) AS Uint8 - ) AS closed_on_this_day -FROM ( - SELECT DISTINCT date_window AS d - FROM `test_results/analytics/tests_monitor` - WHERE date_window >= CurrentUtcDate() - $timeline_days * Interval("P1D") -) AS dt -CROSS JOIN ( - SELECT - t.project_item_id AS project_item_id, - t.issue_id AS issue_id, - t.issue_number AS issue_number, - t.title AS title, - t.url AS url, - t.state AS state, - t.state_reason AS state_reason, - t.created_at AS created_at, - t.updated_at AS updated_at, - t.closed_at AS closed_at, - t.created_date AS created_date, - t.updated_date AS updated_date, - t.author_login AS author_login, - t.author_url AS author_url, - t.repository_name AS repository_name, - t.repository_url AS repository_url, - t.project_status AS project_status, - t.project_owner AS project_owner, - t.project_priority AS project_priority, - t.is_in_project AS is_in_project, - t.days_since_created AS days_since_created, - t.days_since_updated AS days_since_updated, - t.time_to_close_hours AS time_to_close_hours, - t.assignees AS assignees, - t.labels AS labels, - t.milestone AS milestone, - t.project_fields AS project_fields, - t.info AS info, - t.issue_type AS issue_type, - t.exported_at AS exported_at, - COALESCE(m.owner_team, 'unknown') AS owner_team, - CAST(JSON_QUERY(t.labels, "$.name" WITH UNCONDITIONAL ARRAY WRAPPER) AS String) AS labels_list, - COALESCE(JSON_VALUE(t.info, "$.max_branch"), '-') AS max_branch, - COALESCE(JSON_VALUE(t.info, "$.env"), 'env:-') AS env, - COALESCE(JSON_VALUE(t.info, "$.priority"), 'priority:-') AS priority, - COALESCE(JSON_VALUE(t.info, "$.releaseblocker_state"), 'release:-') AS releaseblocker_state, - COALESCE(JSON_VALUE(t.info, "$.branch"), '-') AS branch, - COALESCE(JSON_VALUE(t.info, "$.area"), 'area/-') AS area - FROM `github_data/issues` AS t - LEFT JOIN `test_results/analytics/area_to_owner_mapping` AS m - ON m.area = COALESCE(JSON_VALUE(t.info, "$.area"), 'area/-') - WHERE t.created_date <= CurrentUtcDate() - AND (t.closed_at IS NULL OR Cast(t.closed_at AS Date) >= CurrentUtcDate() - $timeline_days * Interval("P1D")) -) AS i -WHERE i.created_date <= dt.d - AND dt.d >= $month_start AND dt.d < $month_end; +-- For full reload via data_mart_executor.py. Issues on a daily timeline: for each date, which issues are open at end of day and which were closed that day.
+-- Open/closed state and SLA start from github_data/issue_open_periods (exported with issues).
+-- No query CTEs (DataLens-compatible): only scalar params + inline subqueries.
+-- Run: python3 .github/scripts/analytics/data_mart_executor.py --query_path .github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_full.sql --table_path test_results/analytics/github_issues_timeline --store_type column --partition_keys date --primary_keys date issue_number project_item_id --cleanup_window_key date --cleanup_window_interval '365 * Interval("P1D")'
+-- FULL WINDOW: 365-day reload; $month_start/$month_end can be narrowed manually if the query times out.
+--
+$timeline_days = 365;
+$month_start = Date("1970-01-01");
+$month_end = Date("2100-01-01");
+
+SELECT
+ dt.d AS date,
+ i.project_item_id AS project_item_id,
+ i.issue_id AS issue_id,
+ i.issue_number AS issue_number,
+ i.title AS title,
+ i.url AS url,
+ i.state AS state,
+ i.state_reason AS state_reason,
+ i.created_at AS created_at,
+ i.updated_at AS updated_at,
+ i.closed_at AS closed_at,
+ i.created_date AS created_date,
+ i.updated_date AS updated_date,
+ i.author_login AS author_login,
+ i.author_url AS author_url,
+ i.repository_name AS repository_name,
+ i.repository_url AS repository_url,
+ i.project_status AS project_status,
+ i.project_owner AS project_owner,
+ i.project_priority AS project_priority,
+ i.is_in_project AS is_in_project,
+ i.days_since_created AS days_since_created,
+ i.days_since_updated AS days_since_updated,
+ i.time_to_close_hours AS time_to_close_hours,
+ i.assignees AS assignees,
+ i.labels AS labels,
+ i.milestone AS milestone,
+ i.project_fields AS project_fields,
+ i.info AS info,
+ i.issue_type AS issue_type,
+ i.exported_at AS exported_at,
+ i.owner_team AS owner_team,
+ i.labels_list AS labels_list,
+ i.max_branch AS max_branch,
+ i.env AS env,
+ i.priority AS priority,
+ i.releaseblocker_state AS releaseblocker_state,
+ i.branch AS branch,
+ i.area AS area,
+ p.sla_start_date AS sla_start_date,
+ CAST(
+ (p.sla_start_date IS NOT NULL) AS Uint8
+ ) AS is_open_at_end_of_day,
+ CAST(
+ (c.issue_number IS NOT NULL) AS Uint8
+ ) AS closed_on_this_day
+FROM (
+ SELECT DISTINCT date_window AS d
+ FROM `test_results/analytics/tests_monitor`
+ WHERE date_window >= CurrentUtcDate() - $timeline_days * Interval("P1D")
+) AS dt
+CROSS JOIN (
+ SELECT
+ t.project_item_id AS project_item_id,
+ t.issue_id AS issue_id,
+ t.issue_number AS issue_number,
+ t.title AS title,
+ t.url AS url,
+ t.state AS state,
+ t.state_reason AS state_reason,
+ t.created_at AS created_at,
+ t.updated_at AS updated_at,
+ t.closed_at AS closed_at,
+ t.created_date AS created_date,
+ t.updated_date AS updated_date,
+ t.author_login AS author_login,
+ t.author_url AS author_url,
+ t.repository_name AS repository_name,
+ t.repository_url AS repository_url,
+ t.project_status AS project_status,
+ t.project_owner AS project_owner,
+ t.project_priority AS project_priority,
+ t.is_in_project AS is_in_project,
+ t.days_since_created AS days_since_created,
+ t.days_since_updated AS days_since_updated,
+ t.time_to_close_hours AS time_to_close_hours,
+ t.assignees AS assignees,
+ t.labels AS labels,
+ t.milestone AS milestone,
+ t.project_fields AS project_fields,
+ t.info AS info,
+ t.issue_type AS issue_type,
+ t.exported_at AS exported_at,
+ COALESCE(m.owner_team, 'unknown') AS owner_team,
+ CAST(JSON_QUERY(t.labels, "$.name" WITH UNCONDITIONAL ARRAY WRAPPER) AS String) AS labels_list,
+ COALESCE(JSON_VALUE(t.info, "$.max_branch"), '-') AS max_branch,
+ COALESCE(JSON_VALUE(t.info, "$.env"), 'env:-') AS env,
+ COALESCE(JSON_VALUE(t.info, "$.priority"), 'priority:-') AS priority,
+ COALESCE(JSON_VALUE(t.info, "$.releaseblocker_state"), 'release:-') AS releaseblocker_state,
+ COALESCE(JSON_VALUE(t.info, "$.branch"), '-') AS branch,
+ COALESCE(JSON_VALUE(t.info, "$.area"), 'area/-') AS area
+ FROM `github_data/issues` AS t
+ INNER JOIN (
+ SELECT DISTINCT
+ ip.project_item_id AS project_item_id,
+ ip.issue_number AS issue_number
+ FROM (
+ SELECT
+ p.project_item_id AS project_item_id,
+ p.issue_number AS issue_number,
+ p.period_start AS period_start,
+ p.period_end AS period_end
+ FROM `github_data/issue_open_periods` AS p
+ UNION ALL
+ SELECT
+ t2.project_item_id AS project_item_id,
+ t2.issue_number AS issue_number,
+ t2.created_date AS period_start,
+ Cast(t2.closed_at AS Date) AS period_end
+ FROM `github_data/issues` AS t2
+ LEFT JOIN (
+ SELECT DISTINCT
+ ep.project_item_id AS project_item_id,
+ ep.issue_number AS issue_number
+ FROM `github_data/issue_open_periods` AS ep
+ ) AS has_periods
+ ON has_periods.issue_number = t2.issue_number
+ AND has_periods.project_item_id = t2.project_item_id
+ WHERE has_periods.issue_number IS NULL
+ ) AS ip
+ WHERE ip.period_start <= CurrentUtcDate()
+ AND (ip.period_end IS NULL OR ip.period_end >= CurrentUtcDate() - $timeline_days * Interval("P1D"))
+ ) AS w
+ ON w.project_item_id = t.project_item_id AND w.issue_number = t.issue_number
+ LEFT JOIN `test_results/analytics/area_to_owner_mapping` AS m
+ ON m.area = COALESCE(JSON_VALUE(t.info, "$.area"), 'area/-')
+ WHERE t.created_date <= CurrentUtcDate()
+) AS i
+LEFT JOIN (
+ SELECT
+ dt_open.d AS date,
+ ip.project_item_id AS project_item_id,
+ ip.issue_number AS issue_number,
+ ip.period_start AS sla_start_date
+ FROM (
+ SELECT DISTINCT date_window AS d
+ FROM `test_results/analytics/tests_monitor`
+ WHERE date_window >= CurrentUtcDate() - $timeline_days * Interval("P1D")
+ ) AS dt_open
+ CROSS JOIN (
+ SELECT
+ p.project_item_id AS project_item_id,
+ p.issue_number AS issue_number,
+ p.period_start AS period_start,
+ p.period_end AS period_end
+ FROM `github_data/issue_open_periods` AS p
+ UNION ALL
+ SELECT
+ t2.project_item_id AS project_item_id,
+ t2.issue_number AS issue_number,
+ t2.created_date AS period_start,
+ Cast(t2.closed_at AS Date) AS period_end
+ FROM `github_data/issues` AS t2
+ LEFT JOIN (
+ SELECT DISTINCT
+ ep.project_item_id AS project_item_id,
+ ep.issue_number AS issue_number
+ FROM `github_data/issue_open_periods` AS ep
+ ) AS has_periods
+ ON has_periods.issue_number = t2.issue_number
+ AND has_periods.project_item_id = t2.project_item_id
+ WHERE has_periods.issue_number IS NULL
+ ) AS ip
+ WHERE ip.period_start <= dt_open.d
+ AND (ip.period_end IS NULL OR ip.period_end > dt_open.d)
+) AS p
+ ON p.date = dt.d
+ AND p.project_item_id = i.project_item_id
+ AND p.issue_number = i.issue_number
+LEFT JOIN (
+ SELECT DISTINCT
+ ip.period_end AS date,
+ ip.project_item_id AS project_item_id,
+ ip.issue_number AS issue_number
+ FROM (
+ SELECT
+ p.project_item_id AS project_item_id,
+ p.issue_number AS issue_number,
+ p.period_start AS period_start,
+ p.period_end AS period_end
+ FROM `github_data/issue_open_periods` AS p
+ UNION ALL
+ SELECT
+ t2.project_item_id AS project_item_id,
+ t2.issue_number AS issue_number,
+ t2.created_date AS period_start,
+ Cast(t2.closed_at AS Date) AS period_end
+ FROM `github_data/issues` AS t2
+ LEFT JOIN (
+ SELECT DISTINCT
+ ep.project_item_id AS project_item_id,
+ ep.issue_number AS issue_number
+ FROM `github_data/issue_open_periods` AS ep
+ ) AS has_periods
+ ON has_periods.issue_number = t2.issue_number
+ AND has_periods.project_item_id = t2.project_item_id
+ WHERE has_periods.issue_number IS NULL
+ ) AS ip
+ WHERE ip.period_end IS NOT NULL
+) AS c
+ ON c.date = dt.d
+ AND c.project_item_id = i.project_item_id
+ AND c.issue_number = i.issue_number
+WHERE i.created_date <= dt.d
+ AND dt.d >= $month_start AND dt.d < $month_end;
diff --git a/.github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_with_owner_from_mapping.sql b/.github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_with_owner_from_mapping.sql index 35c7888076b..33008eeaf84 100644 --- a/.github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_with_owner_from_mapping.sql +++ b/.github/scripts/analytics/data_mart_queries/datalens_ds_queries/github_issues_timeline_with_owner_from_mapping.sql @@ -40,6 +40,7 @@ SELECT t.max_branch AS max_branch, t.env AS env, t.priority AS priority, + t.releaseblocker_state AS releaseblocker_state, t.branch AS branch, t.area AS area_full, Cast(CASE @@ -54,11 +55,12 @@ SELECT END AS owner_team, t.is_open_at_end_of_day AS is_open_at_end_of_day, t.closed_on_this_day AS closed_on_this_day, + t.sla_start_date AS sla_start_date, CAST( CASE - WHEN t.priority LIKE '%low%' THEN DateTime::ToDays(Cast(t.date AS Date) - Cast(t.created_date AS Date)) < 30 - WHEN t.priority LIKE '%med%' OR t.priority LIKE '%high%' THEN DateTime::ToDays(Cast(t.date AS Date) - Cast(t.created_date AS Date)) < 7 - ELSE DateTime::ToDays(Cast(t.date AS Date) - Cast(t.created_date AS Date)) < 7 + WHEN t.priority LIKE '%low%' THEN DateTime::ToDays(Cast(t.date AS Date) - Cast(COALESCE(t.sla_start_date, t.created_date) AS Date)) < 30 + WHEN t.priority LIKE '%med%' OR t.priority LIKE '%high%' THEN DateTime::ToDays(Cast(t.date AS Date) - Cast(COALESCE(t.sla_start_date, t.created_date) AS Date)) < 7 + ELSE DateTime::ToDays(Cast(t.date AS Date) - Cast(COALESCE(t.sla_start_date, t.created_date) AS Date)) < 7 END AS Uint8 ) AS in_sla FROM `test_results/analytics/github_issues_timeline` AS t diff --git a/.github/scripts/analytics/data_mart_queries/github_issues_bugs_count_by_period.sql b/.github/scripts/analytics/data_mart_queries/github_issues_bugs_count_by_period.sql index 6d46f7ecf87..d0ad1430ca2 100644 --- a/.github/scripts/analytics/data_mart_queries/github_issues_bugs_count_by_period.sql +++ b/.github/scripts/analytics/data_mart_queries/github_issues_bugs_count_by_period.sql @@ -30,6 +30,7 @@ $bugs_raw = ( $normalize(t.area) AS area, t.project_item_id AS project_item_id, t.created_date AS created_date, + t.sla_start_date AS sla_start_date, t.priority AS priority FROM `test_results/analytics/github_issues_timeline` AS t WHERE t.date >= CurrentUtcDate() - $window_days * Interval("P1D") @@ -73,7 +74,7 @@ $bugs = ( b.project_item_id AS project_item_id, b.priority AS priority, o.owner_team AS owner_team, - DateTime::ToDays(Cast(b.date AS Date) - Cast(b.created_date AS Date)) AS days_open + DateTime::ToDays(Cast(b.date AS Date) - Cast(COALESCE(b.sla_start_date, b.created_date) AS Date)) AS days_open FROM $bugs_raw AS b LEFT JOIN $owner AS o ON b.area = o.area ); diff --git a/.github/scripts/analytics/data_mart_queries/github_issues_timeline.sql b/.github/scripts/analytics/data_mart_queries/github_issues_timeline.sql index cbece481272..89aa4c4cfbe 100644 --- a/.github/scripts/analytics/data_mart_queries/github_issues_timeline.sql +++ b/.github/scripts/analytics/data_mart_queries/github_issues_timeline.sql @@ -1,13 +1,13 @@ -- Issues on a daily timeline: for each date, which issues are open at end of day and which were closed that day. --- RECENT DAYS: updates only the last $recent_days days (default 2). Use with data_mart_executor for quick refresh. --- Dates come from tests_monitor (date_window), no ListFromRange/FLATTEN BY. +-- Open/closed state and SLA start from github_data/issue_open_periods (exported with issues). +-- RECENT DAYS: updates only the last $recent_days days (31 by default). Use with data_mart_executor for quick refresh. +-- Dates come from tests_monitor (date_window), no ListFromRange/FLATTEN BY for the date spine. -- In BI: filter by date, owner_team; for issue list per day — filter by date; for counts — GROUP BY date, SUM(is_open_at_end_of_day), SUM(closed_on_this_day). -- $timeline_days = 365; $recent_days = 31; -- only these days are selected (today and $recent_days-1 days back) -- Owner by area (prefix match): area/cs/analytics -> area/cs in mapping. Return matched_area (om.area) for output. --- Distinct areas from source github_data/issues. New areas get owner/area from fallback in output. $owner_mapping = ( SELECT area AS area, owner_team AS owner_team, matched_area AS matched_area FROM ( @@ -23,6 +23,67 @@ $owner_mapping = ( WHERE rn = 1 ); +$issue_periods = ( + SELECT + p.project_item_id AS project_item_id, + p.issue_number AS issue_number, + p.period_start AS period_start, + p.period_end AS period_end + FROM `github_data/issue_open_periods` AS p + UNION ALL + SELECT + t2.project_item_id AS project_item_id, + t2.issue_number AS issue_number, + t2.created_date AS period_start, + Cast(t2.closed_at AS Date) AS period_end + FROM `github_data/issues` AS t2 + LEFT JOIN ( + SELECT DISTINCT + ep.project_item_id AS project_item_id, + ep.issue_number AS issue_number + FROM `github_data/issue_open_periods` AS ep + ) AS has_periods + ON has_periods.issue_number = t2.issue_number + AND has_periods.project_item_id = t2.project_item_id + WHERE has_periods.issue_number IS NULL +); + +$issues_in_window = ( + SELECT DISTINCT + ip.project_item_id AS project_item_id, + ip.issue_number AS issue_number + FROM $issue_periods AS ip + WHERE ip.period_start <= CurrentUtcDate() + AND (ip.period_end IS NULL OR ip.period_end >= CurrentUtcDate() - $timeline_days * Interval("P1D")) +); + +$date_spine = ( + SELECT DISTINCT date_window AS d + FROM `test_results/analytics/tests_monitor` + WHERE date_window >= CurrentUtcDate() - $timeline_days * Interval("P1D") +); + +$open_on_day = ( + SELECT + dt.d AS date, + ip.project_item_id AS project_item_id, + ip.issue_number AS issue_number, + ip.period_start AS sla_start_date + FROM $date_spine AS dt + CROSS JOIN $issue_periods AS ip + WHERE ip.period_start <= dt.d + AND (ip.period_end IS NULL OR ip.period_end > dt.d) +); + +$closed_on_day = ( + SELECT DISTINCT + ip.period_end AS date, + ip.project_item_id AS project_item_id, + ip.issue_number AS issue_number + FROM $issue_periods AS ip + WHERE ip.period_end IS NOT NULL +); + SELECT dt.d AS date, i.project_item_id AS project_item_id, @@ -63,17 +124,14 @@ SELECT i.releaseblocker_state AS releaseblocker_state, i.branch AS branch, i.area AS area, + p.sla_start_date AS sla_start_date, CAST( - (i.closed_at IS NULL OR Cast(i.closed_at AS Date) > dt.d) AS Uint8 + (p.sla_start_date IS NOT NULL) AS Uint8 ) AS is_open_at_end_of_day, CAST( - (i.closed_at IS NOT NULL AND Cast(i.closed_at AS Date) = dt.d) AS Uint8 + (c.issue_number IS NOT NULL) AS Uint8 ) AS closed_on_this_day -FROM ( - SELECT DISTINCT date_window AS d - FROM `test_results/analytics/tests_monitor` - WHERE date_window >= CurrentUtcDate() - $timeline_days * Interval("P1D") -) AS dt +FROM $date_spine AS dt CROSS JOIN ( SELECT t.project_item_id AS project_item_id, @@ -121,9 +179,18 @@ CROSS JOIN ( END ) AS area FROM `github_data/issues` AS t + INNER JOIN $issues_in_window AS w + ON w.project_item_id = t.project_item_id AND w.issue_number = t.issue_number LEFT JOIN $owner_mapping AS m ON m.area = COALESCE(JSON_VALUE(t.info, "$.area"), 'area/-') WHERE t.created_date <= CurrentUtcDate() - AND (t.closed_at IS NULL OR Cast(t.closed_at AS Date) >= CurrentUtcDate() - $timeline_days * Interval("P1D")) ) AS i +LEFT JOIN $open_on_day AS p + ON p.date = dt.d + AND p.project_item_id = i.project_item_id + AND p.issue_number = i.issue_number +LEFT JOIN $closed_on_day AS c + ON c.date = dt.d + AND c.project_item_id = i.project_item_id + AND c.issue_number = i.issue_number WHERE i.created_date <= dt.d AND dt.d >= CurrentUtcDate() - $recent_days * Interval("P1D"); diff --git a/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql b/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql index a56b3574cac..0aa77deae3a 100644 --- a/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql +++ b/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql @@ -132,12 +132,12 @@ SELECT WHEN Db LIKE '%static-node-1.ydb-cluster.com/Root/db%' THEN 'ansible_' WHEN Db LIKE '%ydb-vla-dev04-002%' THEN 'oltp-vla-perf1_' WHEN Db LIKE '%ydb-vla-dev04-005%' THEN 'oltp-vla-perf2_' - WHEN Db LIKE '%ydb-qa-01-klg-010%' THEN 'oltp-klg-perf3_' + WHEN Db LIKE '%ydb-qa-01-klg-015%' THEN 'oltp-klg-perf3_' WHEN Db LIKE '%ydb-qa-01-klg-014%' THEN 'oltp-klg-perf4_' - WHEN Db LIKE '%ydb-qa-01-klg-018%' THEN 'oltp-klg-perf5_' + WHEN Db LIKE '%ydb-qa-01-klg-022%' THEN 'oltp-klg-perf5_' WHEN Db LIKE '%ydb-qa-01-sas-000%' THEN 'oltp-3dc-perf6_' - WHEN Db LIKE '%ydb-qa-01-klg-021%' THEN 'oltp-klg-perf7_' - WHEN Db LIKE '%ydb-qa-01-klg-030%' THEN 'oltp-klg-perf9_' + WHEN Db LIKE '%ydb-qa-01-klg-035%' THEN 'oltp-klg-perf7_' + WHEN Db LIKE '%ydb-qa-01-klg-021%' THEN 'oltp-klg-perf9_' WHEN Db LIKE '%sas%' THEN 'sas_' WHEN Db LIKE '%vla%' THEN 'vla_' WHEN Db LIKE '%klg%' THEN 'klg_' diff --git a/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql b/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql index 9eb51df5577..a09a80d0ee1 100644 --- a/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql +++ b/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql @@ -141,12 +141,12 @@ SELECT WHEN s.Db LIKE '%static-node-1.ydb-cluster.com/Root/db%' THEN 'ansible_' WHEN s.Db LIKE '%ydb-vla-dev04-002%' THEN 'oltp-vla-perf1_' WHEN s.Db LIKE '%ydb-vla-dev04-005%' THEN 'oltp-vla-perf2_' - WHEN s.Db LIKE '%ydb-qa-01-klg-010%' THEN 'oltp-klg-perf3_' + WHEN s.Db LIKE '%ydb-qa-01-klg-015%' THEN 'oltp-klg-perf3_' WHEN s.Db LIKE '%ydb-qa-01-klg-014%' THEN 'oltp-klg-perf4_' - WHEN s.Db LIKE '%ydb-qa-01-klg-018%' THEN 'oltp-klg-perf5_' + WHEN s.Db LIKE '%ydb-qa-01-klg-022%' THEN 'oltp-klg-perf5_' WHEN s.Db LIKE '%ydb-qa-01-sas-000%' THEN 'oltp-3dc-perf6_' - WHEN s.Db LIKE '%ydb-qa-01-klg-021%' THEN 'oltp-klg-perf7_' - WHEN s.Db LIKE '%ydb-qa-01-klg-030%' THEN 'oltp-klg-perf9_' + WHEN s.Db LIKE '%ydb-qa-01-klg-035%' THEN 'oltp-klg-perf7_' + WHEN s.Db LIKE '%ydb-qa-01-klg-021%' THEN 'oltp-klg-perf9_' WHEN s.Db LIKE '%sas%' THEN 'sas_' WHEN s.Db LIKE '%vla%' THEN 'vla_' WHEN s.Db LIKE '%klg%' THEN 'klg_' diff --git a/.github/scripts/analytics/export_issues_to_ydb.py b/.github/scripts/analytics/export_issues_to_ydb.py index 0dcbe6a04ce..05baecae122 100755 --- a/.github/scripts/analytics/export_issues_to_ydb.py +++ b/.github/scripts/analytics/export_issues_to_ydb.py @@ -8,7 +8,7 @@ import json import argparse from datetime import datetime, timezone, timedelta import requests -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Tuple from ydb_wrapper import YDBWrapper ORG_NAME = 'ydb-platform' @@ -165,8 +165,9 @@ def fetch_single_issue(org_name: str, repo_name: str, issue_number: int) -> Opti issueType { name } - timelineItems(last: 20, itemTypes: [CLOSED_EVENT]) { + timelineItems(last: 50, itemTypes: [CLOSED_EVENT, REOPENED_EVENT]) { nodes { + __typename ... on ClosedEvent { createdAt actor { @@ -174,6 +175,9 @@ def fetch_single_issue(org_name: str, repo_name: str, issue_number: int) -> Opti login } } + ... on ReopenedEvent { + createdAt + } } } projectItems(first: 30) { @@ -287,8 +291,9 @@ def fetch_repository_issues(org_name: str = ORG_NAME, repo_name: str = REPO_NAME issueType { name } - timelineItems(last: 20, itemTypes: [CLOSED_EVENT]) { + timelineItems(last: 50, itemTypes: [CLOSED_EVENT, REOPENED_EVENT]) { nodes { + __typename ... on ClosedEvent { createdAt actor { @@ -296,6 +301,9 @@ def fetch_repository_issues(org_name: str = ORG_NAME, repo_name: str = REPO_NAME login } } + ... on ReopenedEvent { + createdAt + } } } projectItems(first: 30) { @@ -507,7 +515,7 @@ def extract_last_close_actor(issue: Dict[str, Any]) -> Dict[str, Any]: event_at = None nodes = (issue.get('timelineItems') or {}).get('nodes') or [] for event in reversed(nodes): - if not event: + if not event or event.get('__typename') != 'ClosedEvent': continue actor = event.get('actor') or {} cand_login = actor.get('login') or '' @@ -519,6 +527,86 @@ def extract_last_close_actor(issue: Dict[str, Any]) -> Dict[str, Any]: return {'login': login, 'actor_type': actor_type, 'event_at': event_at} +def build_open_periods( + issue: Dict[str, Any], + created_at: Optional[datetime], + closed_at: Optional[datetime], +) -> List[Dict[str, Optional[str]]]: + """Build open intervals from GitHub close/reopen timeline events. + + Each period is ``{"start": "YYYY-MM-DD", "end": "YYYY-MM-DD"|null}``. + ``end`` is the close date (issue is open through end-of-day before ``end``). + """ + if created_at is None: + return [] + + events: List[tuple[str, datetime]] = [] + for event in (issue.get('timelineItems') or {}).get('nodes') or []: + if not event: + continue + typename = event.get('__typename') or '' + event_at = parse_datetime(event.get('createdAt')) + if event_at is None: + continue + if typename == 'ClosedEvent': + events.append(('close', event_at)) + elif typename == 'ReopenedEvent': + events.append(('reopen', event_at)) + + events.sort(key=lambda item: item[1]) + + periods: List[Dict[str, Optional[str]]] = [] + open_start = created_at + is_open = True + + for kind, event_at in events: + if kind == 'close' and is_open: + periods.append({ + 'start': open_start.date().isoformat(), + 'end': event_at.date().isoformat(), + }) + is_open = False + elif kind == 'reopen' and not is_open: + open_start = event_at + is_open = True + + if is_open: + end = None + if (issue.get('state') or '').upper() == 'CLOSED' and closed_at is not None: + end = closed_at.date().isoformat() + periods.append({ + 'start': open_start.date().isoformat(), + 'end': end, + }) + + return periods + + +def open_period_rows_for_issue( + issue_number: int, + project_item_id: str, + open_periods: List[Dict[str, Optional[str]]], + exported_at: datetime, +) -> List[Dict[str, Any]]: + rows = [] + for idx, period in enumerate(open_periods): + period_start = datetime.fromisoformat(period['start']).date() + period_end = ( + datetime.fromisoformat(period['end']).date() + if period.get('end') + else None + ) + rows.append({ + 'issue_number': issue_number, + 'project_item_id': project_item_id, + 'period_index': idx, + 'period_start': period_start, + 'period_end': period_end, + 'exported_at': exported_at, + }) + return rows + + def projects_for_info_json(issue: Dict[str, Any]) -> List[Dict[str, Any]]: """Projects (v2) that contain this issue — id/title from GraphQL ``projectItems``.""" out = [] @@ -577,8 +665,11 @@ def get_max_branch(branch_labels): return best -def transform_issues_for_ydb(issues: List[Dict[str, Any]], project_fields: Optional[Dict[int, Dict[str, Any]]] = None) -> List[Dict[str, Any]]: - """Transform GitHub issues data for YDB storage""" +def transform_issues_for_ydb(issues: List[Dict[str, Any]], project_fields: Optional[Dict[int, Dict[str, Any]]] = None) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Transform GitHub issues data for YDB storage. + + Returns (issue_records, open_period_rows) for github_data/issues and github_data/issue_open_periods. + """ print("Transforming issues data for YDB...") start_time = time.time() @@ -586,6 +677,7 @@ def transform_issues_for_ydb(issues: List[Dict[str, Any]], project_fields: Optio project_fields = {} transformed_issues = [] + open_period_rows: List[Dict[str, Any]] = [] for issue in issues: # Get project fields for this issue if available @@ -687,6 +779,8 @@ def transform_issues_for_ydb(issues: List[Dict[str, Any]], project_fields: Optio if closed_at: info['closed_at_iso'] = closed_at.isoformat() + open_periods = build_open_periods(issue, created_at, closed_at) + now = datetime.now(timezone.utc) is_in_project = bool(issue_project_fields) @@ -703,7 +797,6 @@ def transform_issues_for_ydb(issues: List[Dict[str, Any]], project_fields: Optio if closed_at and created_at: time_to_close_hours = int((closed_at - created_at).total_seconds() / 3600) - # Build the record issue_record = { # Primary identifiers 'project_item_id': f"repo-{issue.get('number', 0)}", @@ -758,10 +851,18 @@ def transform_issues_for_ydb(issues: List[Dict[str, Any]], project_fields: Optio } transformed_issues.append(issue_record) + open_period_rows.extend( + open_period_rows_for_issue( + issue_record['issue_number'], + issue_record['project_item_id'], + open_periods, + now, + ) + ) elapsed = time.time() - start_time - print(f"Transformed {len(transformed_issues)} issues (took {elapsed:.2f}s)") - return transformed_issues + print(f"Transformed {len(transformed_issues)} issues, {len(open_period_rows)} open periods (took {elapsed:.2f}s)") + return transformed_issues, open_period_rows def create_issues_table(ydb_wrapper: YDBWrapper, table_path: str): """Create issues table in YDB optimized for BI""" @@ -838,6 +939,80 @@ def create_issues_table(ydb_wrapper: YDBWrapper, table_path: str): elapsed = time.time() - start_time print(f"BI-optimized table created successfully (took {elapsed:.2f}s)") + +def create_issue_open_periods_table(ydb_wrapper: YDBWrapper, table_path: str): + """Open/close intervals per issue for timeline and SLA.""" + print(f"Creating issue open periods table: {table_path}") + create_sql = f""" + CREATE TABLE IF NOT EXISTS `{table_path}` ( + `issue_number` Uint64 NOT NULL, + `project_item_id` Utf8 NOT NULL, + `period_index` Uint32 NOT NULL, + `period_start` Date NOT NULL, + `period_end` Date, + `exported_at` Timestamp NOT NULL, + PRIMARY KEY (`issue_number`, `project_item_id`, `period_index`) + ) + PARTITION BY HASH(`issue_number`) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_BY_SIZE = ENABLED, + AUTO_PARTITIONING_PARTITION_SIZE_MB = 2048, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 4 + ) + """ + ydb_wrapper.create_table(table_path, create_sql) + + +def delete_issue_open_periods(ydb_wrapper: YDBWrapper, table_path: str, issue_numbers: List[int]) -> None: + if not issue_numbers: + return + for issue_number in issue_numbers: + query = f""" + DECLARE $issue_number AS Uint64; + DELETE FROM `{table_path}` WHERE issue_number = $issue_number; + """ + ydb_wrapper.execute_dml( + query, + { + '$issue_number': ydb.TypedValue( + value=int(issue_number), + value_type=ydb.PrimitiveType.Uint64, + ), + }, + query_name='delete_issue_open_periods', + ) + + +def upload_issue_open_periods( + ydb_wrapper: YDBWrapper, + table_path: str, + period_rows: List[Dict[str, Any]], + issue_numbers: List[int], + batch_size: int = 500, +) -> None: + create_issue_open_periods_table(ydb_wrapper, table_path) + delete_issue_open_periods(ydb_wrapper, table_path, issue_numbers) + if not period_rows: + return + column_types = ( + ydb.BulkUpsertColumns() + .add_column("issue_number", ydb.OptionalType(ydb.PrimitiveType.Uint64)) + .add_column("project_item_id", ydb.OptionalType(ydb.PrimitiveType.Utf8)) + .add_column("period_index", ydb.OptionalType(ydb.PrimitiveType.Uint32)) + .add_column("period_start", ydb.OptionalType(ydb.PrimitiveType.Date)) + .add_column("period_end", ydb.OptionalType(ydb.PrimitiveType.Date)) + .add_column("exported_at", ydb.OptionalType(ydb.PrimitiveType.Timestamp)) + ) + ydb_wrapper.bulk_upsert_batches( + table_path, + period_rows, + column_types, + batch_size, + query_name='issue_open_periods', + ) + print(f"Uploaded {len(period_rows)} rows to {table_path}") + def main(): """Main function to export GitHub issues to YDB""" parser = argparse.ArgumentParser(description='Export GitHub issues to YDB') @@ -864,6 +1039,7 @@ def main(): # Get table path from config table_path = ydb_wrapper.get_table_path("issues") + periods_table_path = ydb_wrapper.get_table_path("issue_open_periods") batch_size = 100 try: @@ -923,7 +1099,7 @@ def main(): project_fields = get_project_fields_for_issues(ORG_NAME, PROJECT_ID, issue_numbers) # Transform issues for YDB - transformed_issues = transform_issues_for_ydb(issues, project_fields) + transformed_issues, open_period_rows = transform_issues_for_ydb(issues, project_fields) # Upsert issues in batches using bulk_upsert_batches print(f"Uploading {len(transformed_issues)} issues in batches of {batch_size}") @@ -1011,6 +1187,15 @@ def main(): ) ydb_wrapper.bulk_upsert_batches(table_path, transformed_issues, column_types, batch_size) + + issue_numbers = [row['issue_number'] for row in transformed_issues] + upload_issue_open_periods( + ydb_wrapper, + periods_table_path, + open_period_rows, + issue_numbers, + batch_size=500, + ) script_elapsed = time.time() - script_start_time print(f"Script completed successfully (total time: {script_elapsed:.2f}s)") diff --git a/.github/scripts/analytics/flaky_tests_history.py b/.github/scripts/analytics/flaky_tests_history.py index e3b3338059a..ce14e4e5bd9 100755 --- a/.github/scripts/analytics/flaky_tests_history.py +++ b/.github/scripts/analytics/flaky_tests_history.py @@ -21,7 +21,8 @@ def _dedupe_history_rows(rows): BASE_DATE = datetime.date(1970, 1, 1) -DEFAULT_MONTHS_BACK = 180 # 6 months +DEFAULT_DAYS_BACK = 30 # cold start when branch has no flaky_tests_window history +MAX_RESUME_GAP_DAYS = 180 # safety cap on incremental resume if collection was paused a long time def parse_arguments(): @@ -89,12 +90,17 @@ def determine_start_date(ydb_wrapper, test_runs_table, flaky_tests_table, build_ Logic: 1. If start_date_override is provided, use it (must be <= end_date) - 2. If history exists, use max_date_window from history (but not after end_date) - 3. If no history, check test_runs_table for min_run_date - 4. Default to 6 months ago from end_date + 2. If history exists, continue from max(date_window) (incremental; no cold-start cap), + but never look back further than MAX_RESUME_GAP_DAYS even if collection was + paused for a long time — avoids re-creating the unbounded-backfill problem this + cold-start rework was meant to fix, at the cost of leaving an un-backfilled gap + in the rare case collection was dormant that long. + 3. If no history, check test_runs_table for min_run_date, capped at DEFAULT_DAYS_BACK + 4. Default cold start to DEFAULT_DAYS_BACK ago from end_date """ end_date = end_date_override if end_date_override else datetime.date.today() - default_start_date = end_date - datetime.timedelta(days=DEFAULT_MONTHS_BACK) + default_start_date = end_date - datetime.timedelta(days=DEFAULT_DAYS_BACK) + max_resume_start_date = end_date - datetime.timedelta(days=MAX_RESUME_GAP_DAYS) # If user explicitly provided start_date_override, use it if start_date_override: @@ -106,13 +112,16 @@ def determine_start_date(ydb_wrapper, test_runs_table, flaky_tests_table, build_ max_date_window = get_max_date_from_history(ydb_wrapper, flaky_tests_table, build_type, branch) if max_date_window is not None: - # Clamp max_date_window to not exceed end_date before comparison - max_date_clamped = min(max_date_window, end_date) - # Use max_date_clamped if it's greater than default_start_date - if max_date_clamped > default_start_date: - start_date = max_date_clamped - else: - start_date = default_start_date + # Incremental resume: continue from the last stored day (do not apply cold-start + # lookback), capped so a long-dormant branch cannot trigger an unbounded backfill. + start_date = min(max_date_window, end_date) + if start_date < max_resume_start_date: + print( + f'⚠️ Last recorded date_window ({start_date}) is more than {MAX_RESUME_GAP_DAYS} days ' + f'old — history collection looks paused. Capping resume start to ' + f'{max_resume_start_date} instead of backfilling the full gap.' + ) + start_date = max_resume_start_date return start_date, end_date # No history exists, check test_runs_table for min date diff --git a/.github/scripts/analytics/tests_monitor.py b/.github/scripts/analytics/tests_monitor.py index 151ec444cff..de424b48a96 100755 --- a/.github/scripts/analytics/tests_monitor.py +++ b/.github/scripts/analytics/tests_monitor.py @@ -18,6 +18,8 @@ from github_issue_utils import ( ) from testowners_utils import normalize_github_team_owners_string +NEW_BRANCH_MONITOR_LOOKBACK_DAYS = 30 + def _dedupe_monitor_rows(rows): """One row per (full_name, date_window, branch, build_type); prefer deepest suite_folder.""" @@ -527,11 +529,25 @@ def main(): branch_creation_date = None if branch_creation_date: - process_start_date = max(branch_creation_date, default_start_date) - print(f"Found branch creation date: {branch_creation_date}") + cold_start_date = max( + today - datetime.timedelta(days=NEW_BRANCH_MONITOR_LOOKBACK_DAYS), + default_start_date, + ) + process_start_date = max(branch_creation_date, cold_start_date) + print( + f"Found branch creation date: {branch_creation_date}, " + f"collecting from {process_start_date} " + f"(lookback capped at {NEW_BRANCH_MONITOR_LOOKBACK_DAYS} days)" + ) else: - process_start_date = max(today - datetime.timedelta(days=7), default_start_date) - print(f"No test runs found for branch, using 1 week ago: {process_start_date}") + process_start_date = max( + today - datetime.timedelta(days=NEW_BRANCH_MONITOR_LOOKBACK_DAYS), + default_start_date, + ) + print( + f"No test runs found for branch, collecting from {process_start_date} " + f"({NEW_BRANCH_MONITOR_LOOKBACK_DAYS} days ago)" + ) date_list = [process_start_date + datetime.timedelta(days=x) for x in range((today - process_start_date).days + 1)] print(f"Init new monitor collecting from date {process_start_date}") diff --git a/.github/scripts/cherry_pick_v2.py b/.github/scripts/cherry_pick_v2.py index f1a4aee609d..6efa79cb906 100755 --- a/.github/scripts/cherry_pick_v2.py +++ b/.github/scripts/cherry_pick_v2.py @@ -56,6 +56,10 @@ class BackportResult: return len(self.conflict_files) > 0 +class ChangesAlreadyAppliedError(Exception): + """Raised when all requested commits are already present in the target branch.""" + + def run_git(repo_path: str, cmd: List[str], logger, check=True) -> subprocess.CompletedProcess: """Run git command""" result = subprocess.run( @@ -132,6 +136,15 @@ def create_pr_source(pull: Any, allow_unmerged: bool, logger) -> Source: ) +def is_empty_cherry_pick(output: str) -> bool: + """Detects git empty cherry-pick when patch is already present in the branch.""" + output_lower = output.lower() + return ( + 'now empty' in output_lower + or 'nothing added to commit' in output_lower + ) + + def detect_conflicts(repo_path: str, logger) -> List[ConflictInfo]: """Detects conflicts from git status""" conflict_files = [] @@ -494,6 +507,17 @@ def process_branch( output = (result.stdout or '') + (('\n' + result.stderr) if result.stderr else '') if result.returncode != 0: + if is_empty_cherry_pick(output): + run_git(repo_path, ['cherry-pick', '--skip'], logger, check=False) + logger.info( + "Commit %s is already present in %s, skipping", + commit_sha[:7], target_branch + ) + cherry_pick_logs.append( + f"=== Cherry-picking {commit_sha[:7]} ===\n" + f"Skipped: changes already present in branch\n{output}" + ) + continue if "conflict" in output.lower(): conflicts = detect_conflicts(repo_path, logger) if conflicts: @@ -510,6 +534,20 @@ def process_branch( cherry_pick_logs.append(f"=== Cherry-picking {commit_sha[:7]} ===\n{output}") except subprocess.CalledProcessError as e: raise RuntimeError(f"Cherry-pick failed for commit {commit_sha[:7]}: {e}") + + ahead_result = run_git( + repo_path, ['rev-list', '--count', f'{target_branch}..HEAD'], logger, check=False + ) + if ahead_result.returncode != 0: + raise RuntimeError( + f"Failed to count commits ahead of {target_branch}: " + f"{(ahead_result.stderr or ahead_result.stdout or '').strip()}" + ) + commits_ahead = int((ahead_result.stdout or '0').strip() or 0) + if commits_ahead == 0: + raise ChangesAlreadyAppliedError( + f"All requested changes are already present in {target_branch}" + ) # Push branch run_git(repo_path, ['push', '--set-upstream', 'origin', dev_branch_name], logger) @@ -781,6 +819,12 @@ def main(): repo_name, repo, token, sources, workflow_triggerer, workflow_url, summary_path, logger ) results.append(result) + except ChangesAlreadyAppliedError as e: + logger.info("Branch %s skipped: %s", target_branch, e) + if summary_path: + with open(summary_path, 'a') as f: + f.write(f"Branch `{target_branch}`: skipped (changes already present)\n\n") + skipped_branches.append((target_branch, "changes already present")) except Exception as e: has_errors = True error_msg = f"UNEXPECTED_ERROR: Branch {target_branch} - {type(e).__name__}: {e}" diff --git a/.github/scripts/tests/mute/constants.py b/.github/scripts/tests/mute/constants.py index 1bf059abf8e..218e578e7f1 100644 --- a/.github/scripts/tests/mute/constants.py +++ b/.github/scripts/tests/mute/constants.py @@ -14,6 +14,7 @@ _REQUIRED_KEYS = ( 'mute_window_days', 'unmute_window_days', 'delete_window_days', + 'stable_branch_grace_days', 'manual_unmute_issue_closed_lookback_days', 'manual_unmute_currently_muted_lookback_days', ) @@ -68,6 +69,12 @@ def get_delete_window_days(): return _positive_int('delete_window_days') +def get_stable_branch_grace_days(): + """Calendar days a newly added stable branch keeps inherited ``muted_ya`` lines + (protected from zero-run delete) before normal mute/unmute/delete rules take over.""" + return _positive_int('stable_branch_grace_days') + + def get_manual_unmute_issue_closed_lookback_days(): return _positive_int('manual_unmute_issue_closed_lookback_days') diff --git a/.github/scripts/tests/mute/create_new_muted_ya.py b/.github/scripts/tests/mute/create_new_muted_ya.py index 1cf8c86e919..03c341e1f39 100755 --- a/.github/scripts/tests/mute/create_new_muted_ya.py +++ b/.github/scripts/tests/mute/create_new_muted_ya.py @@ -4,6 +4,7 @@ import datetime import json import os import re +import subprocess import ydb import logging import sys @@ -33,6 +34,7 @@ from mute.constants import ( get_manual_unmute_min_runs, get_manual_unmute_window_days, get_mute_window_days, + get_stable_branch_grace_days, get_unmute_window_days, ) from mute.naming import mute_file_line_to_tests_monitor_full_name @@ -54,6 +56,193 @@ _DIGEST_NOTIFICATION_CONFIG = os.path.normpath( os.path.join(dir, '..', '..', '..', 'config', 'mute_issue_and_digest_config.json') ) +_STABLE_BRANCHES_CONFIG = '.github/config/stable_tests_branches.json' + + +def _grace_inherited_debug_line(line, branch, config_since, grace_until): + return ( + f"{line} # GRACE: inherited mute ({branch}, config since " + f"{config_since.isoformat()}, until {grace_until.isoformat()}, no monitor data yet)" + ) + + +def _git_branch_added_to_stable_config(branch, repo_root): + """Calendar date the branch first appeared in ``_STABLE_BRANCHES_CONFIG``. + + ``git log -S<needle>`` (pickaxe) narrows to the commit(s) that changed the + branch string's occurrence count, instead of ``git show``-ing every commit + that ever touched the config file on every scheduled run. + """ + if not branch or branch == 'main': + return None + try: + proc = subprocess.run( + [ + 'git', 'log', '--format=%H', '--reverse', '-S' + json.dumps(branch), + '--', _STABLE_BRANCHES_CONFIG, + ], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + logging.warning( + 'stable branch grace: git log failed for branch=%s: %s', + branch, + exc, + ) + return None + if proc.returncode != 0: + logging.warning( + 'stable branch grace: git log exit %s for %s: %s', + proc.returncode, + _STABLE_BRANCHES_CONFIG, + (proc.stderr or '').strip(), + ) + return None + for commit in proc.stdout.splitlines(): + commit = commit.strip() + if not commit: + continue + show = subprocess.run( + ['git', 'show', f'{commit}:{_STABLE_BRANCHES_CONFIG}'], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + if show.returncode != 0: + logging.warning( + 'stable branch grace: git show %s:%s failed with exit %s: %s', + commit, + _STABLE_BRANCHES_CONFIG, + show.returncode, + (show.stderr or '').strip(), + ) + continue + try: + branches = json.loads(show.stdout) + names = {str(b).strip() for b in branches if str(b).strip()} + except (json.JSONDecodeError, TypeError): + continue + if branch not in names: + continue + dproc = subprocess.run( + ['git', 'log', '-1', '--format=%aI', commit], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + if dproc.returncode != 0: + logging.warning( + 'stable branch grace: git log -1 date for commit %s failed with exit %s: %s', + commit, + dproc.returncode, + (dproc.stderr or '').strip(), + ) + continue + raw = dproc.stdout.strip() + if raw.endswith('Z'): + raw = raw[:-1] + '+00:00' + try: + return datetime.datetime.fromisoformat(raw).astimezone(datetime.timezone.utc).date() + except ValueError: + logging.warning( + 'stable branch grace: invalid author date for commit %s: %r', + commit, + raw, + ) + continue + return None + + +def _debug_line_test_string(debug_line): + """Recover the raw ``testsuite testcase`` prefix from a ``create_debug_string`` line.""" + return debug_line.split(' # ', 1)[0] + + +def _apply_stable_branch_grace( + branch, + inherited_muted_ya_path, + all_muted_ya, + all_muted_ya_debug, + to_delete, + to_delete_debug, + repo_root, +): + """Keep inherited ``muted_ya`` lines for a new stable branch during its grace window. + + Returns ``(all_muted_ya, all_muted_ya_debug, to_delete, to_delete_debug, + grace_inherited, grace_config_since, grace_until)``. The ``*_debug`` lists are + kept 1:1 with their raw counterparts, so grace can't desync ``foo.txt`` from + ``foo_debug.txt`` (previously it could, e.g. a test grace protected from + deletion would still show up in ``to_delete_debug.txt`` as removed). + """ + inactive = all_muted_ya, all_muted_ya_debug, to_delete, to_delete_debug, frozenset(), None, None + added = _git_branch_added_to_stable_config(branch, repo_root) + if added is None: + return inactive + today = datetime.datetime.now(datetime.timezone.utc).date() + grace_days = get_stable_branch_grace_days() + grace_until = added + datetime.timedelta(days=grace_days - 1) + if today > grace_until: + return inactive + try: + with open(inherited_muted_ya_path, encoding='utf-8') as fp: + inherited = {line.strip() for line in fp if line.strip()} + except OSError as exc: + logging.warning( + 'stable branch grace: cannot read inherited mute file %s: %s', + inherited_muted_ya_path, + exc, + ) + return inactive + if not inherited: + return inactive + + # to_delete: drop debug lines for entries grace just pulled back out of to_delete + # (matched by their raw "testsuite testcase" prefix, so wildcard delete patterns — + # whose debug line is rendered from one concrete chunk rather than the pattern + # itself — are conservatively left as-is rather than mismatched). + removed_from_delete = set(to_delete) & inherited + new_to_delete = sorted(set(to_delete) - inherited) + new_to_delete_debug = sorted( + d for d in to_delete_debug if _debug_line_test_string(d) not in removed_from_delete + ) + + # muted_ya: add a synthetic debug line for every inherited entry that grace newly + # restores. all_muted_ya/all_muted_ya_debug are 1:1 going in, so anything not + # already in all_muted_ya cannot already have a debug line either. + newly_added = inherited - set(all_muted_ya) + grace_debug_lines = [ + _grace_inherited_debug_line(line, branch, added, grace_until) for line in sorted(newly_added) + ] + new_all_muted_ya = sorted(set(all_muted_ya) | inherited) + new_all_muted_ya_debug = sorted(list(all_muted_ya_debug) + grace_debug_lines) + + logging.info( + 'stable branch grace for %s (config since %s, until %s): keep %d inherited mute(s) ' + '(%d newly restored, %d protected from zero-run delete)', + branch, + added, + grace_until, + len(inherited), + len(newly_added), + len(removed_from_delete), + ) + return ( + new_all_muted_ya, + new_all_muted_ya_debug, + new_to_delete, + new_to_delete_debug, + frozenset(inherited), + added, + grace_until, + ) + + def load_manual_unmute_config(): """Manual fast-unmute window — required keys in ``mute_config.json`` via ``mute.constants``.""" return get_manual_unmute_window_days(), get_manual_unmute_min_runs() @@ -711,6 +900,8 @@ def apply_and_add_mutes( ydb_wrapper=None, branch=None, build_type=None, + inherited_muted_ya_path=None, + repo_root=None, ): output_path = os.path.join(output_path, 'mute_update') logging.info(f"Creating mute files in directory: {output_path}") @@ -872,12 +1063,33 @@ def apply_and_add_mutes( to_delete = sorted(list(set(to_delete) | set(manual_fast_delete_lines))) to_delete_debug = sorted(list(set(to_delete_debug) | set(manual_fast_delete_debug))) - write_file_set(os.path.join(output_path, 'to_delete.txt'), to_delete, to_delete_debug) - # 4. muted_ya (all currently muted tests). all_muted_ya, all_muted_ya_debug = create_file_set( all_data, lambda test: mute_check(test.get('suite_folder'), test.get('test_name')) if mute_check else True, use_wildcards=True, resolution='muted_ya' ) + grace_inherited = frozenset() + grace_config_since = None + grace_until = None + if branch and inherited_muted_ya_path and repo_root: + ( + all_muted_ya, + all_muted_ya_debug, + to_delete, + to_delete_debug, + grace_inherited, + grace_config_since, + grace_until, + ) = _apply_stable_branch_grace( + branch, + inherited_muted_ya_path, + all_muted_ya, + all_muted_ya_debug, + to_delete, + to_delete_debug, + repo_root, + ) + + write_file_set(os.path.join(output_path, 'to_delete.txt'), to_delete, to_delete_debug) write_file_set(os.path.join(output_path, 'muted_ya.txt'), all_muted_ya, all_muted_ya_debug) to_mute_set = set(to_mute) to_unmute_set = set(to_unmute) @@ -896,6 +1108,15 @@ def apply_and_add_mutes( if is_chunk_test(test): wildcard_key = create_test_string(test, use_wildcards=True) wildcard_to_chunks[wildcard_key].append(test) + for line in grace_inherited: + if ( + line not in test_debug_dict + and grace_config_since is not None + and grace_until is not None + ): + test_debug_dict[line] = _grace_inherited_debug_line( + line, branch, grace_config_since, grace_until + ) # Build wildcard-level debug strings. for wildcard, chunks in wildcard_to_chunks.items(): N = len(chunks) @@ -1554,6 +1775,8 @@ def mute_worker(args): ydb_wrapper=ydb_wrapper, branch=args.branch, build_type=build_type, + inherited_muted_ya_path=input_muted_ya_path, + repo_root=repo_path.rstrip(os.sep), ) elif args.mode == 'sync_fast_unmute_grace': diff --git a/.github/scripts/tests/mute/tests/__init__.py b/.github/scripts/tests/mute/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/.github/scripts/tests/mute/tests/__init__.py diff --git a/.github/scripts/tests/mute/tests/test_flaky_tests_history_resume.py b/.github/scripts/tests/mute/tests/test_flaky_tests_history_resume.py new file mode 100644 index 00000000000..816d985bd38 --- /dev/null +++ b/.github/scripts/tests/mute/tests/test_flaky_tests_history_resume.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3
+"""Tests for the cold-start / incremental-resume logic in ``flaky_tests_history.py``.
+
+Run from ``.github/scripts/tests/mute``: ``python3 -m unittest discover -s tests``
+(running from the repo root shadows the installed ``ydb`` package).
+"""
+import datetime
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+_HERE = Path(__file__).resolve().parent
+_MUTE_DIR = _HERE.parent
+_TESTS_DIR = _MUTE_DIR.parent
+_SCRIPTS_DIR = _TESTS_DIR.parent
+for _p in (str(_TESTS_DIR), str(_SCRIPTS_DIR), str(_SCRIPTS_DIR / 'analytics')):
+ if _p not in sys.path:
+ sys.path.insert(0, _p)
+
+import flaky_tests_history as fth # noqa: E402
+
+
+class DetermineStartDateTest(unittest.TestCase):
+ def setUp(self):
+ self.ydb_wrapper = mock.Mock()
+ self.end_date = datetime.date(2026, 7, 14)
+
+ def _call(self, start_date_override=None, end_date_override=None):
+ return fth.determine_start_date(
+ self.ydb_wrapper,
+ test_runs_table='test_runs',
+ flaky_tests_table='flaky_tests_window',
+ build_type='relwithdebinfo',
+ branch='stable-x',
+ start_date_override=start_date_override,
+ end_date_override=end_date_override,
+ )
+
+ def test_explicit_override_wins(self):
+ override = datetime.date(2026, 6, 1)
+ start, end = self._call(start_date_override=override, end_date_override=self.end_date)
+ self.assertEqual(start, override)
+ self.assertEqual(end, self.end_date)
+
+ def test_recent_history_resumes_without_gap(self):
+ # Normal steady-state case: last recorded day is yesterday -> resume exactly
+ # there (no artificial cold-start-style jump, no gap).
+ recent = self.end_date - datetime.timedelta(days=1)
+ with mock.patch.object(fth, 'get_max_date_from_history', return_value=recent):
+ start, end = self._call(end_date_override=self.end_date)
+ self.assertEqual(start, recent)
+ self.assertEqual(end, self.end_date)
+
+ def test_moderately_stale_history_still_resumes_from_last_day(self):
+ # A gap smaller than MAX_RESUME_GAP_DAYS must not be capped — this is exactly
+ # the "avoid history gaps" behavior the PR intentionally introduced.
+ stale = self.end_date - datetime.timedelta(days=fth.MAX_RESUME_GAP_DAYS - 10)
+ with mock.patch.object(fth, 'get_max_date_from_history', return_value=stale):
+ start, end = self._call(end_date_override=self.end_date)
+ self.assertEqual(start, stale)
+
+ def test_very_stale_history_is_capped_not_unbounded(self):
+ # Collection paused for ~2 years: resuming from the true last-recorded day
+ # would re-create the unbounded-backfill problem this rework was meant to
+ # avoid. Must be capped to MAX_RESUME_GAP_DAYS instead.
+ ancient = self.end_date - datetime.timedelta(days=730)
+ with mock.patch.object(fth, 'get_max_date_from_history', return_value=ancient):
+ start, end = self._call(end_date_override=self.end_date)
+ expected = self.end_date - datetime.timedelta(days=fth.MAX_RESUME_GAP_DAYS)
+ self.assertEqual(start, expected)
+ self.assertGreater(start, ancient)
+
+ def test_cold_start_uses_min_run_date_capped_at_default_days_back(self):
+ very_old_run = self.end_date - datetime.timedelta(days=400)
+ with mock.patch.object(fth, 'get_max_date_from_history', return_value=None), \
+ mock.patch.object(fth, 'get_min_date_from_test_runs', return_value=very_old_run):
+ start, end = self._call(end_date_override=self.end_date)
+ self.assertEqual(start, self.end_date - datetime.timedelta(days=fth.DEFAULT_DAYS_BACK))
+
+ def test_cold_start_no_data_anywhere_uses_default_days_back(self):
+ with mock.patch.object(fth, 'get_max_date_from_history', return_value=None), \
+ mock.patch.object(fth, 'get_min_date_from_test_runs', return_value=None):
+ start, end = self._call(end_date_override=self.end_date)
+ self.assertEqual(start, self.end_date - datetime.timedelta(days=fth.DEFAULT_DAYS_BACK))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/.github/scripts/tests/mute/tests/test_stable_branch_grace.py b/.github/scripts/tests/mute/tests/test_stable_branch_grace.py new file mode 100644 index 00000000000..377678f1e27 --- /dev/null +++ b/.github/scripts/tests/mute/tests/test_stable_branch_grace.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3
+"""Tests for the stable-branch-grace feature in ``create_new_muted_ya.py``: git-history
+date lookup, ``*_debug`` list parity with their raw counterparts, and the on-disk files.
+
+Run from ``.github/scripts/tests/mute``: ``python3 -m unittest discover -s tests``
+(running from the repo root shadows the installed ``ydb`` package).
+"""
+import datetime
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+_HERE = Path(__file__).resolve().parent
+_MUTE_DIR = _HERE.parent
+_TESTS_DIR = _MUTE_DIR.parent
+_SCRIPTS_DIR = _TESTS_DIR.parent
+for _p in (str(_MUTE_DIR), str(_TESTS_DIR), str(_SCRIPTS_DIR), str(_SCRIPTS_DIR / 'analytics')):
+ if _p not in sys.path:
+ sys.path.insert(0, _p)
+
+import create_new_muted_ya as cnm # noqa: E402
+
+
+def _git(args, cwd, env=None):
+ subprocess.run(['git', *args], cwd=cwd, check=True, capture_output=True, env=env)
+
+
+class _GitConfigRepoMixin:
+ """Builds a throwaway git repo with a commit history for stable_tests_branches.json."""
+
+ def setUp(self):
+ self._tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self._tmp.name)
+ self.config_rel = cnm._STABLE_BRANCHES_CONFIG
+ self.config_path = self.root / self.config_rel
+ self.config_path.parent.mkdir(parents=True)
+ _git(['init', '-q'], cwd=self.root)
+ _git(['config', 'user.email', '[email protected]'], cwd=self.root)
+ _git(['config', 'user.name', 'Test User'], cwd=self.root)
+
+ def tearDown(self):
+ self._tmp.cleanup()
+
+ def _commit_config(self, branches, date_iso):
+ self.config_path.write_text(json.dumps(branches, indent=4) + '\n', encoding='utf-8')
+ env = os.environ.copy()
+ env['GIT_AUTHOR_DATE'] = date_iso
+ env['GIT_COMMITTER_DATE'] = date_iso
+ _git(['add', self.config_rel], cwd=self.root)
+ _git(['commit', '-q', '-m', f'update branches: {branches}'], cwd=self.root, env=env)
+
+ def _write_inherited(self, lines):
+ path = self.root / 'muted_ya.txt'
+ path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
+ return str(path)
+
+
+class GitBranchAddedToStableConfigTest(_GitConfigRepoMixin, unittest.TestCase):
+ def test_first_seen_from_git_history(self):
+ self._commit_config(['main'], '2026-01-01T12:00:00+00:00')
+ self._commit_config(['main', 'stable-26-3'], '2026-07-10T10:00:00+00:00')
+
+ added = cnm._git_branch_added_to_stable_config('stable-26-3', str(self.root))
+ self.assertEqual(added, datetime.date(2026, 7, 10))
+
+ def test_main_is_never_dated(self):
+ self._commit_config(['main', 'stable-26-3'], '2026-07-10T10:00:00+00:00')
+ self.assertIsNone(cnm._git_branch_added_to_stable_config('main', str(self.root)))
+
+ def test_branch_never_added_returns_none(self):
+ self._commit_config(['main', 'stable-26-3'], '2026-07-10T10:00:00+00:00')
+ added = cnm._git_branch_added_to_stable_config('stable-does-not-exist', str(self.root))
+ self.assertIsNone(added)
+
+ def test_no_branch_argument_returns_none(self):
+ self.assertIsNone(cnm._git_branch_added_to_stable_config('', str(self.root)))
+ self.assertIsNone(cnm._git_branch_added_to_stable_config(None, str(self.root)))
+
+ def test_multiple_config_edits_still_finds_first_addition(self):
+ # Several unrelated edits before and after the branch is added must not
+ # confuse the pickaxe fast path into picking the wrong commit.
+ self._commit_config(['main'], '2026-01-01T00:00:00+00:00')
+ self._commit_config(['main', 'stable-25-3'], '2026-02-01T00:00:00+00:00')
+ self._commit_config(['main', 'stable-25-3', 'stable-25-4'], '2026-03-01T00:00:00+00:00')
+ self._commit_config(
+ ['main', 'stable-25-3', 'stable-25-4', 'stable-26-1'], '2026-04-01T00:00:00+00:00'
+ )
+ self._commit_config(
+ ['main', 'stable-25-4', 'stable-26-1'], '2026-05-01T00:00:00+00:00'
+ ) # stable-25-3 removed
+
+ self.assertEqual(
+ cnm._git_branch_added_to_stable_config('stable-26-1', str(self.root)),
+ datetime.date(2026, 4, 1),
+ )
+ self.assertEqual(
+ cnm._git_branch_added_to_stable_config('stable-25-4', str(self.root)),
+ datetime.date(2026, 3, 1),
+ )
+
+ def test_missing_repo_root_does_not_raise(self):
+ # No git repo at all at this path: git will fail (non-zero exit), the function
+ # must degrade to None rather than raising.
+ added = cnm._git_branch_added_to_stable_config('stable-26-3', str(self.root / 'nope'))
+ self.assertIsNone(added)
+
+
+def _debug_prefixes(debug_lines):
+ return {cnm._debug_line_test_string(d) for d in debug_lines}
+
+
+class ApplyStableBranchGraceTest(_GitConfigRepoMixin, unittest.TestCase):
+ def test_inactive_when_branch_never_added(self):
+ self._commit_config(['main'], '2026-01-01T00:00:00+00:00')
+ inherited_path = self._write_inherited(['suite testA'])
+ result = cnm._apply_stable_branch_grace(
+ 'stable-unknown', inherited_path, ['x'], ['x # debug'], ['y'], ['y # debug'], str(self.root)
+ )
+ all_muted_ya, all_muted_ya_debug, to_delete, to_delete_debug, grace_inherited, since, until = result
+ self.assertEqual(all_muted_ya, ['x'])
+ self.assertEqual(to_delete, ['y'])
+ self.assertEqual(grace_inherited, frozenset())
+ self.assertIsNone(since)
+ self.assertIsNone(until)
+
+ def test_inactive_after_grace_window_expires(self):
+ self._commit_config(['main', 'stable-26-3'], '2020-01-01T00:00:00+00:00')
+ inherited_path = self._write_inherited(['suite testA'])
+ result = cnm._apply_stable_branch_grace(
+ 'stable-26-3', inherited_path, [], [], ['suite testA'], ['suite testA # debug'], str(self.root)
+ )
+ all_muted_ya, all_muted_ya_debug, to_delete, to_delete_debug, grace_inherited, since, until = result
+ # Long expired -> untouched passthrough.
+ self.assertEqual(to_delete, ['suite testA'])
+ self.assertEqual(grace_inherited, frozenset())
+
+ def test_active_grace_protects_delete_and_keeps_debug_counts_in_sync(self):
+ today = datetime.datetime.now(datetime.timezone.utc).date()
+ self._commit_config(['main', 'stable-26-3'], today.isoformat() + 'T00:00:00+00:00')
+
+ # "suiteA testA" has zero monitor runs -> normally a delete candidate, and is
+ # also present in the inherited file, so grace must protect it.
+ # "suiteB testB" is inherited but never showed up in monitor data at all (the
+ # classic brand-new-branch case) -> grace must add it to muted_ya from scratch.
+ inherited_path = self._write_inherited(['suiteA testA', 'suiteB testB'])
+
+ all_muted_ya = ['suiteA testA']
+ all_muted_ya_debug = ['suiteA testA # owner o success_rate 0% [2026-01-01], p-0, f-0,m-0, s-0, runs-0, mute state: muted, test state: Muted']
+ to_delete = ['suiteA testA']
+ to_delete_debug = list(all_muted_ya_debug)
+
+ (
+ new_all_muted_ya,
+ new_all_muted_ya_debug,
+ new_to_delete,
+ new_to_delete_debug,
+ grace_inherited,
+ since,
+ until,
+ ) = cnm._apply_stable_branch_grace(
+ 'stable-26-3', inherited_path, all_muted_ya, all_muted_ya_debug, to_delete, to_delete_debug,
+ str(self.root),
+ )
+
+ self.assertEqual(since, today)
+ self.assertEqual(until, today + datetime.timedelta(days=cnm.get_stable_branch_grace_days() - 1))
+ self.assertEqual(grace_inherited, frozenset({'suiteA testA', 'suiteB testB'}))
+
+ # Bug regression check #1: to_delete must no longer contain the protected test,
+ # and to_delete_debug must be trimmed to match (same length, same test coverage).
+ self.assertEqual(new_to_delete, [])
+ self.assertEqual(len(new_to_delete), len(new_to_delete_debug))
+ self.assertEqual(new_to_delete_debug, [])
+
+ # Bug regression check #2: muted_ya gains suiteB testB, and muted_ya_debug must
+ # gain a matching line for it too (line counts must stay equal).
+ self.assertEqual(new_all_muted_ya, ['suiteA testA', 'suiteB testB'])
+ self.assertEqual(len(new_all_muted_ya), len(new_all_muted_ya_debug))
+ self.assertEqual(_debug_prefixes(new_all_muted_ya_debug), {'suiteA testA', 'suiteB testB'})
+ # The pre-existing debug line for suiteA testA (real monitor data) must be
+ # preserved untouched, not replaced by the generic grace placeholder.
+ self.assertIn(all_muted_ya_debug[0], new_all_muted_ya_debug)
+
+ def test_grace_does_not_resurrect_legitimate_unmute(self):
+ # Grace only protects against zero-run delete; it must not fight the unmute
+ # rule when a test genuinely has enough passing runs on the new branch.
+ today = datetime.datetime.now(datetime.timezone.utc).date()
+ self._commit_config(['main', 'stable-26-3'], today.isoformat() + 'T00:00:00+00:00')
+ inherited_path = self._write_inherited(['suiteA testA'])
+
+ all_muted_ya = ['suiteA testA']
+ all_muted_ya_debug = ['suiteA testA # owner o success_rate 100% [2026-01-01], p-5, f-0,m-0, s-0, runs-5, mute state: muted, test state: Active']
+ to_delete = []
+ to_delete_debug = []
+
+ result = cnm._apply_stable_branch_grace(
+ 'stable-26-3', inherited_path, all_muted_ya, all_muted_ya_debug, to_delete, to_delete_debug,
+ str(self.root),
+ )
+ new_all_muted_ya = result[0]
+ # all_muted_ya still contains it (grace only ever adds/keeps); the caller's
+ # separate to_unmute computation (outside this function) is what actually
+ # drops it from the final muted_ya-to_unmute output.
+ self.assertEqual(new_all_muted_ya, ['suiteA testA'])
+
+
+class ApplyAndAddMutesEndToEndTest(_GitConfigRepoMixin, unittest.TestCase):
+ """Exercise apply_and_add_mutes() end-to-end and check the files written to disk."""
+
+ def _read_lines(self, path):
+ if not os.path.exists(path):
+ return []
+ with open(path, encoding='utf-8') as fp:
+ return [line.rstrip('\n') for line in fp]
+
+ def test_to_delete_and_muted_ya_files_stay_paired_with_their_debug_files(self):
+ today = datetime.datetime.now(datetime.timezone.utc).date()
+ self._commit_config(['main', 'stable-x'], today.isoformat() + 'T00:00:00+00:00')
+
+ inherited_path = self._write_inherited(['suiteA testA', 'suiteB testB'])
+
+ all_data = [
+ {
+ 'test_name': 'testA', 'suite_folder': 'suiteA', 'full_name': 'suiteA/testA',
+ 'build_type': 'relwithdebinfo', 'branch': 'stable-x',
+ 'date_window': today,
+ 'pass_count': 0, 'fail_count': 0, 'mute_count': 0, 'skip_count': 0,
+ 'owner': 'o', 'is_muted': True, 'state': 'Muted', 'days_in_state': 1,
+ 'is_test_chunk': 0,
+ },
+ ]
+ aggregated_for_delete = [dict(all_data[0], period_days=7)]
+
+ out_dir = self.root / 'out'
+ out_dir.mkdir()
+
+ cnm.apply_and_add_mutes(
+ all_data,
+ str(out_dir),
+ mute_check=lambda suite, case: True,
+ aggregated_for_mute=[],
+ aggregated_for_unmute=[],
+ aggregated_for_delete=aggregated_for_delete,
+ branch='stable-x',
+ build_type='relwithdebinfo',
+ inherited_muted_ya_path=inherited_path,
+ repo_root=str(self.root),
+ )
+
+ mute_update_dir = out_dir / 'mute_update'
+ to_delete = self._read_lines(mute_update_dir / 'to_delete.txt')
+ to_delete_debug = self._read_lines(mute_update_dir / 'to_delete_debug.txt')
+ muted_ya = self._read_lines(mute_update_dir / 'muted_ya.txt')
+ muted_ya_debug = self._read_lines(mute_update_dir / 'muted_ya_debug.txt')
+
+ # Grace must have protected suiteA/testA from deletion and pulled in suiteB/testB.
+ self.assertEqual(to_delete, [])
+ self.assertEqual(sorted(muted_ya), ['suiteA testA', 'suiteB testB'])
+
+ # The actual bug this test guards against: line counts (and thus the
+ # "Removed from mute" / muted_ya PR-body sections built from *_debug.txt)
+ # must stay 1:1 with their .txt counterparts.
+ self.assertEqual(len(to_delete), len(to_delete_debug))
+ self.assertEqual(len(muted_ya), len(muted_ya_debug))
+
+
+if __name__ == '__main__':
+ unittest.main()
|