CI: Postcommit gate manual control wf (HOTFIX) (#19069)

2 files changed, 453 insertions, 0 deletions

diff --git a/.github/workflows/gate_postcommits_manual_control.yml b/.github/workflows/gate_postcommits_manual_control.yml
new file mode 100644
index 00000000000..fb06ba82fd7
--- /dev/null
+++ b/.github/workflows/gate_postcommits_manual_control.yml
@@ -0,0 +1,224 @@
+name: Manual Gate Control
+on:
+  workflow_dispatch:
+    inputs:
+      action:
+        description: 'Action to perform'
+        required: true
+        default: 'open'
+        type: choice
+        options:
+          - open
+          - close
+      count_of_runners_to_change_label:
+        description: 'Optional: Number of runners to change (leave empty for all)'
+        required: false
+        type: string
+
+jobs:
+  manage_gate:
+    runs-on: ubuntu-latest
+    name: Manual Gate Control
+    steps:
+    - name: Set Gate State
+      shell: bash
+      run: |
+        echo "Performing manual gate ${{ github.event.inputs.action }} operation"
+        
+        # Set limit count if provided
+        LIMIT_COUNT="${{ github.event.inputs.count_of_runners_to_change_label }}"
+        if [[ -n "$LIMIT_COUNT" && "$LIMIT_COUNT" =~ ^[0-9]+$ ]]; then
+          echo "Will change labels for up to $LIMIT_COUNT runners"
+          LIMIT_OPTION=true
+        else
+          echo "Will change labels for all applicable runners"
+          LIMIT_OPTION=false
+        fi
+        
+        if [[ "${{ github.event.inputs.action }}" == "close" ]]; then
+          echo "Closing gate for postcommits - removing postcommit labels"
+          query='.runners[] | select(.labels[].name=="ghrun") | select( any([.labels[].name][]; .=="postcommit")) | .id'
+          OPERATION="Closing"
+          LABEL_ACTION="Removing"
+        else
+          echo "Opening gate for postcommits - adding postcommit labels"
+          query='.runners[] | select(.labels[].name=="ghrun") | select( all([.labels[].name][]; .!="postcommit")) | .id'
+          OPERATION="Opening"
+          LABEL_ACTION="Adding"
+        fi
+        
+        # Function to fetch a page of runners
+        fetch_runners_page() {
+          local page=$1
+          echo "Fetching runners page $page..."
+          local result=$(curl -Ls -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{secrets.GH_PERSONAL_ACCESS_TOKEN}}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" -w "%{http_code}\n" \
+            "https://api.github.com/repos/${{github.repository}}/actions/runners?per_page=100&page=$page")
+          
+          local http_code=$(echo "$result" | tail -n 1)
+          if [ "$http_code" != "200" ]; then
+            echo "HTTP error fetching page $page: $http_code"
+            echo "$result"
+            return 1
+          fi
+          
+          # Remove status code from the end
+          echo "$result" | sed '$d'
+        }
+        
+        # Get all runners with pagination
+        echo "Collecting all runners (paginated)..."
+        page=1
+        all_runner_ids=()
+        more_pages=true
+        
+        while $more_pages; do
+          runners_data=$(fetch_runners_page $page)
+          if [ $? -ne 0 ]; then
+            echo "Failed to fetch runners page $page, exiting"
+            exit 1
+          fi
+          
+          # Extract runner IDs for this page
+          readarray -t page_ids < <(echo "$runners_data" | jq -r "$query" | grep -v "^$")
+          
+          # Check if we have runners on this page
+          count_on_page=${#page_ids[@]}
+          echo "Found $count_on_page eligible runners on page $page"
+          
+          # Append to our array
+          all_runner_ids+=("${page_ids[@]}")
+          
+          # Check if we have more pages
+          runners_count=$(echo "$runners_data" | jq '.runners | length')
+          if [ $runners_count -lt 100 ]; then
+            more_pages=false
+          else
+            page=$((page + 1))
+          fi
+        done
+        
+        # Count total eligible runners
+        total_runners=${#all_runner_ids[@]}
+        echo "Total eligible runners found across all pages: $total_runners"
+        
+        # Limit if needed
+        if [[ "$LIMIT_OPTION" == "true" && $total_runners -gt $LIMIT_COUNT ]]; then
+          all_runner_ids=("${all_runner_ids[@]:0:$LIMIT_COUNT}")
+          echo "Limited to first $LIMIT_COUNT runners"
+        fi
+        
+        # Files to store results and details
+        success_file=$(mktemp)
+        failed_file=$(mktemp)
+        details_file=$(mktemp)
+        
+        # Process runners
+        changed_count=0
+        for runner_id in "${all_runner_ids[@]}"; do
+          changed_count=$((changed_count + 1))
+          echo "Processing runner $changed_count/${#all_runner_ids[@]}: ID $runner_id"
+          
+          if [[ "${{ github.event.inputs.action }}" == "close" ]]; then
+            echo "Removing postcommit label from runner $runner_id"
+            response=$(curl -s -w "\n%{http_code}" -X DELETE -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${{secrets.GH_PERSONAL_ACCESS_TOKEN}}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id/labels/postcommit")
+          else
+            echo "Adding postcommit label to runner $runner_id"
+            response=$(curl -s -w "\n%{http_code}" -X POST -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${{secrets.GH_PERSONAL_ACCESS_TOKEN}}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              "https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id/labels" \
+              -d '{"labels":["postcommit"]}')
+          fi
+          
+          # Parse response
+          http_code=$(echo "$response" | tail -n1)
+          body=$(echo "$response" | sed '$d')
+          
+          # Get runner name for better reporting
+          runner_name=$(curl -s -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{secrets.GH_PERSONAL_ACCESS_TOKEN}}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id" | \
+            jq -r '.name // "Unknown"')
+          
+          # Check success (HTTP 2xx)
+          if [[ $http_code -ge 200 && $http_code -lt 300 ]]; then
+            echo "1" >> "$success_file"
+            status_icon="✅"
+            status_text="Success"
+            echo "$status_icon Success: Label ${{ github.event.inputs.action == 'close' && 'removed' || 'added' }} for runner $runner_name (ID: $runner_id)"
+          else
+            echo "1" >> "$failed_file"
+            status_icon="❌"
+            status_text="Failed"
+            echo "$status_icon Failed: HTTP $http_code for runner $runner_name (ID: $runner_id)"
+            echo "Error response: $body"
+          fi
+          
+          # Add to details for summary
+          echo "| $runner_name | $runner_id | $status_text | $http_code |" >> "$details_file"
+        done
+        
+        # Calculate success/failure counts
+        success_count=$(wc -l < "$success_file")
+        failed_count=$(wc -l < "$failed_file")
+        
+        # Create summary for GitHub Actions
+        echo "## Gate Control Operation Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "**Action:** $OPERATION gate ($LABEL_ACTION 'postcommit' label)" >> $GITHUB_STEP_SUMMARY
+        echo "**Repository:** \`${{github.repository}}\`" >> $GITHUB_STEP_SUMMARY
+        echo "**Timestamp:** $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        
+        echo "### Results" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY
+        echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
+        echo "| Total eligible runners | $total_runners |" >> $GITHUB_STEP_SUMMARY
+        if [[ "$LIMIT_OPTION" == "true" ]]; then
+          echo "| Limit applied | $LIMIT_COUNT |" >> $GITHUB_STEP_SUMMARY
+        fi
+        echo "| Runners processed | $changed_count |" >> $GITHUB_STEP_SUMMARY
+        echo "| ✅ Successful operations | $success_count |" >> $GITHUB_STEP_SUMMARY
+        echo "| ❌ Failed operations | $failed_count |" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        
+        if [[ $changed_count -gt 0 ]]; then
+          echo "### Operation Details" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Runner Name | ID | Status | HTTP Code |" >> $GITHUB_STEP_SUMMARY
+          echo "|-------------|----|---------|----|" >> $GITHUB_STEP_SUMMARY
+          cat "$details_file" >> $GITHUB_STEP_SUMMARY
+        else
+          echo "### No runners were processed" >> $GITHUB_STEP_SUMMARY
+        fi
+        
+        # Console summary
+        echo ""
+        echo "=== OPERATION SUMMARY ==="
+        echo "Action: ${{ github.event.inputs.action }} gate"
+        echo "Total eligible runners: $total_runners"
+        if [[ "$LIMIT_OPTION" == "true" ]]; then
+          echo "Limited to: $LIMIT_COUNT"
+        fi
+        echo "Runners processed: $changed_count"
+        echo "Successful operations: $success_count"
+        echo "Failed operations: $failed_count"
+        echo "========================="
+        
+        # Cleanup temp files
+        rm -f "$success_file" "$failed_file" "$details_file"
+        
+        # Set exit code based on success
+        if [[ $failed_count -gt 0 ]]; then
+          echo "Warning: Some operations failed"
+          echo "::warning::$failed_count operations failed. See summary for details."
+          exit 1
+        fi
+        
+        echo "Gate ${{ github.event.inputs.action }} operation completed successfully"
diff --git a/ydb/ci/debug/get_status.py b/ydb/ci/debug/get_status.py
new file mode 100644
index 00000000000..0d74a6bcb16
--- /dev/null
+++ b/ydb/ci/debug/get_status.py
@@ -0,0 +1,229 @@
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+import time
+import os
+import sys
+from datetime import datetime
+from collections import defaultdict
+from prettytable import PrettyTable
+
+GITHUB_TOKEN = os.getenv('GITHUB_TOKEN')
+if not GITHUB_TOKEN:
+    print("Ошибка: Не установлена переменная окружения GITHUB_TOKEN")
+    print("Установите её перед запуском скрипта:")
+    print("export GITHUB_TOKEN='ваш_токен'")
+    sys.exit(1)
+OWNER = "ydb-platform"
+REPO = "ydb"
+
+# Настройка сессии с retry-стратегией
+session = requests.Session()
+retry_strategy = Retry(
+    total=3,
+    backoff_factor=1,
+    status_forcelist=[429, 500, 502, 503, 504],
+)
+adapter = HTTPAdapter(max_retries=retry_strategy)
+session.mount("https://", adapter)
+session.mount("http://", adapter)
+
+headers = {
+    "Authorization": f"token {GITHUB_TOKEN}",
+    "Accept": "application/vnd.github+json"
+}
+
+WORKFLOW_STATUSES = ["queued", "in_progress", "waiting"]
+JOB_STATUSES = ["queued", "in_progress", "waiting", "pending", "started"]
+
+def make_request(url, params=None):
+    """Выполнить HTTP-запрос с обработкой ошибок"""
+    max_attempts = 3
+    attempt = 0
+    while attempt < max_attempts:
+        try:
+            response = session.get(url, headers=headers, params=params, timeout=30)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            attempt += 1
+            if attempt == max_attempts:
+                print(f"Ошибка при запросе {url}: {str(e)}")
+                return None
+            print(f"Попытка {attempt} из {max_attempts} не удалась. Повторная попытка через {attempt * 2} секунд...")
+            time.sleep(attempt * 2)
+
+def get_workflows():
+    """Получить список всех workflows в репозитории"""
+    print("\nПолучаем список workflows...")
+    url = f"https://api.github.com/repos/{OWNER}/{REPO}/actions/workflows"
+    params = {"per_page": 100}
+    
+    response = make_request(url, params)
+    if response:
+        workflows = response.get("workflows", [])
+        print(f"Найдено {len(workflows)} workflows")
+        return workflows
+    return []
+
+def get_workflow_runs(workflow_id, workflow_name):
+    """Получить runs для конкретного workflow"""
+    print(f"\nПолучаем runs для workflow '{workflow_name}'...")
+    url = f"https://api.github.com/repos/{OWNER}/{REPO}/actions/workflows/{workflow_id}/runs"
+    params = {"per_page": 100}
+    
+    response = make_request(url, params)
+    if response:
+        runs = response.get("workflow_runs", [])
+        active_runs = [run for run in runs if run["status"] in WORKFLOW_STATUSES]
+        print(f"Найдено {len(active_runs)} активных runs")
+        return active_runs
+    return []
+
+def get_run_jobs(run_id, workflow_name):
+    """Получить jobs для конкретного run"""
+    print(f"  Получаем jobs для run {run_id} (workflow: '{workflow_name}')...")
+    url = f"https://api.github.com/repos/{OWNER}/{REPO}/actions/runs/{run_id}/jobs"
+    params = {"per_page": 100}
+    
+    response = make_request(url, params)
+    if response:
+        jobs = response.get("jobs", [])
+        print(f"  Найдено {len(jobs)} jobs")
+        return jobs
+    return []
+
+def get_pr_check_details():
+    """Получить детальную информацию по PR-check"""
+    print("\nПолучаем детальную информацию по PR-check...")
+    
+    workflows = get_workflows()
+    pr_check_workflow = next((wf for wf in workflows if wf["name"] == "PR-check"), None)
+    
+    if not pr_check_workflow:
+        print("PR-check workflow не найден")
+        return
+    
+    runs = get_workflow_runs(pr_check_workflow["id"], "PR-check")
+    grouped_jobs = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
+    
+    for run in runs:
+        jobs = get_run_jobs(run["id"], "PR-check")
+        run_date = datetime.strptime(run["created_at"], "%Y-%m-%dT%H:%M:%SZ")
+        
+        for job in jobs:
+            if job["status"] in JOB_STATUSES:
+                job_info = {
+                    "job_url": job["html_url"],
+                    "job_status": job["status"],
+                    "run_status": run["status"],
+                    "run_url": run["html_url"],
+                    "created_at": run_date,
+                    "run_id": run["id"]
+                }
+                grouped_jobs[job["name"]][job["status"]][run_date.date()].append(job_info)
+        
+        time.sleep(0.5)
+    
+    # Детальная таблица
+    detail_table = PrettyTable()
+    detail_table.field_names = ["Job Name", "Status", "Date", "Job URL", "Run Status", "Run URL"]
+    detail_table.max_width = 50
+    detail_table.align = "l"
+    
+    for job_name in sorted(grouped_jobs.keys()):
+        for status in JOB_STATUSES:
+            if status in grouped_jobs[job_name]:
+                for date in sorted(grouped_jobs[job_name][status].keys()):
+                    for job_info in grouped_jobs[job_name][status][date]:
+                        detail_table.add_row([
+                            job_name[:47] + "..." if len(job_name) > 50 else job_name,
+                            status,
+                            date.strftime("%Y-%m-%d"),
+                            job_info["job_url"],
+                            job_info["run_status"],
+                            job_info["run_url"]
+                        ])
+    
+    print("\nДетальная информация по PR-check:")
+    print(detail_table)
+    
+    # Сводная статистика
+    summary = defaultdict(lambda: defaultdict(int))
+    for job_name in grouped_jobs:
+        for status in grouped_jobs[job_name]:
+            for date in grouped_jobs[job_name][status]:
+                summary[job_name][status] += len(grouped_jobs[job_name][status][date])
+    
+    summary_table = PrettyTable()
+    summary_table.field_names = ["Job Name"] + JOB_STATUSES
+    summary_table.align = "l"
+    
+    for job_name in sorted(summary.keys()):
+        row = [job_name[:47] + "..." if len(job_name) > 50 else job_name]
+        for status in JOB_STATUSES:
+            row.append(summary[job_name][status])
+        summary_table.add_row(row)
+    
+    print("\nСводная статистика по PR-check jobs:")
+    print(summary_table)
+
+def generate_summary_tables():
+    """Генерация сводных таблиц по всем workflows и jobs"""
+    workflows = get_workflows()
+    if not workflows:
+        print("Нет доступных workflows")
+        return
+
+    workflow_stats = defaultdict(lambda: {status: 0 for status in WORKFLOW_STATUSES})
+    job_stats = defaultdict(lambda: {status: 0 for status in JOB_STATUSES})
+
+    for i, wf in enumerate(workflows, 1):
+        print(f"\nОбработка workflow {i}/{len(workflows)}: '{wf['name']}'")
+        runs = get_workflow_runs(wf["id"], wf["name"])
+        
+        for run in runs:
+            workflow_stats[wf["name"]][run["status"]] += 1
+            
+            jobs = get_run_jobs(run["id"], wf["name"])
+            for job in jobs:
+                if job["status"] in JOB_STATUSES:
+                    job_stats[wf["name"]][job["status"]] += 1
+            
+            time.sleep(0.5)
+
+    # Таблица workflows
+    workflow_table = PrettyTable()
+    workflow_table.field_names = ["Workflow Name"] + WORKFLOW_STATUSES
+    
+    # Таблица jobs
+    job_table = PrettyTable()
+    job_table.field_names = ["Workflow Name"] + JOB_STATUSES
+
+    # Заполнение таблиц
+    for wf_name in workflow_stats:
+        stats = workflow_stats[wf_name]
+        if sum(stats.values()) > 0:
+            workflow_table.add_row([wf_name] + [stats[status] for status in WORKFLOW_STATUSES])
+        
+        job_stat = job_stats[wf_name]
+        if sum(job_stat.values()) > 0:
+            job_table.add_row([wf_name] + [job_stat[status] for status in JOB_STATUSES])
+
+    print("\nСводка по активным workflow runs:")
+    print(workflow_table)
+    print("\nСводка по активным jobs:")
+    print(job_table)
+    
+    # Детальная информация по PR-check
+    get_pr_check_details()
+
+def main():
+    try:
+        print("Сбор статистики по активным workflow runs и jobs...")
+        generate_summary_tables()
+    except Exception as e:
+        print(f"Произошла непредвиденная ошибка: {str(e)}")
+
+if __name__ == "__main__":
+    main()