Upload error statistics (#13783)

2 files changed, 67 insertions, 29 deletions

diff --git a/ydb/tests/olap/lib/ydb_cli.py b/ydb/tests/olap/lib/ydb_cli.py
index 1e8334fe699..382af12f1dd 100644
--- a/ydb/tests/olap/lib/ydb_cli.py
+++ b/ydb/tests/olap/lib/ydb_cli.py
@@ -45,6 +45,26 @@ class YdbCliHelper:
             self.ast = ast
             self.svg = svg
 
+    class Iteration:
+        def __init__(self):
+            self.plan: Optional[YdbCliHelper.QueryPlan] = None
+            self.error_message: Optional[str] = None
+            self.time: Optional[float] = None
+
+        def get_error_class(self) -> str:
+            msg_to_class = {
+                'Deadline Exceeded': 'timeout',
+                'Request timeout': 'timeout',
+                'Query did not complete within specified timeout': 'timeout',
+                'There is diff': 'diff'
+            }
+            for msg, cl in msg_to_class.items():
+                if self.error_message and self.error_message.find(msg) >= 0:
+                    return cl
+            if self.error_message:
+                return 'other'
+            return ''
+
     class WorkloadRunResult:
         def __init__(self):
             self.stats: dict[str, dict[str, Any]] = {}
@@ -54,9 +74,8 @@ class YdbCliHelper:
             self.error_message: str = ''
             self.warning_message: str = ''
             self.plans: Optional[list[YdbCliHelper.QueryPlan]] = None
-            self.explain_plan: Optional[YdbCliHelper.QueryPlan] = None
-            self.errors_by_iter: dict[int, str] = {}
-            self.time_by_iter: dict[int, float] = {}
+            self.explain = YdbCliHelper.Iteration()
+            self.iterations: dict[int, YdbCliHelper.Iteration] = {}
             self.traceback: Optional[TracebackType] = None
             self.start_time = time()
 
@@ -64,6 +83,18 @@ class YdbCliHelper:
         def success(self) -> bool:
             return len(self.error_message) == 0
 
+        def get_error_stats(self):
+            result = {}
+            for iter in self.iterations.values():
+                cl = iter.get_error_class()
+                if cl:
+                    result[cl] = True
+            if len(result) == 0 and self.error_message:
+                result['other'] = True
+            if self.warning_message:
+                result['warning'] = True
+            return result
+
     class WorkloadProcessor:
         def __init__(self,
                      workload_type: WorkloadType,
@@ -107,6 +138,10 @@ class YdbCliHelper:
                 else:
                     self.result.warning_message = msg
 
+        def _init_iter(self, iter_num: int) -> None:
+            if iter_num not in self.result.iterations:
+                self.result.iterations[iter_num] = YdbCliHelper.Iteration()
+
         def _process_returncode(self, returncode) -> None:
             begin_str = f'{self.query_num}:'
             end_str = 'Query text:'
@@ -127,9 +162,10 @@ class YdbCliHelper:
                         begin_pos = end_pos + 1
                     end_pos = self.result.stderr.find(end_str, begin_pos)
                     msg = (self.result.stderr[begin_pos:] if end_pos < 0 else self.result.stderr[begin_pos:end_pos]).strip()
-                    self.result.errors_by_iter[iter] = msg
+                    self._init_iter(iter)
+                    self.result.iterations[iter].error_message = msg
                     self._add_error(f'Iteration {iter}: {msg}')
-            if returncode != 0 and len(self.result.errors_by_iter) == 0:
+            if returncode != 0 and len(filter(lambda x: x.error_message, self.result.iterations.values())) == 0:
                 self._add_error(f'Invalid return code: {returncode} instead 0.')
 
         def _load_plan(self, name: str) -> YdbCliHelper.QueryPlan:
@@ -150,8 +186,10 @@ class YdbCliHelper:
             return result
 
         def _load_plans(self) -> None:
-            self.result.plans = [self._load_plan(str(i)) for i in range(self.iterations)]
-            self.result.explain_plan = self._load_plan('explain')
+            for i in range(self.iterations):
+                self._init_iter(i)
+                self.result.iterations[i].plan = self._load_plan(str(i))
+            self.result.explain.plan = self._load_plan('explain')
 
         def _load_stats(self):
             if not os.path.exists(self._json_path):
@@ -199,7 +237,9 @@ class YdbCliHelper:
             for line in self.result.stdout.splitlines():
                 m = re.search(r'iteration ([0-9]*):\s*ok\s*([\.0-9]*)s', line)
                 if m is not None:
-                    self.result.time_by_iter[int(m.group(1))] = float(m.group(2))
+                    iter = int(m.group(1))
+                    self._init_iter(iter)
+                    self.result.iterations[iter].time = float(m.group(2))
 
         def _get_cmd(self) -> list[str]:
             cmd = YdbCliHelper.get_cli_command() + [
diff --git a/ydb/tests/olap/load/lib/conftest.py b/ydb/tests/olap/load/lib/conftest.py
index 915338e378b..a93cc8511ad 100644
--- a/ydb/tests/olap/load/lib/conftest.py
+++ b/ydb/tests/olap/load/lib/conftest.py
@@ -169,31 +169,27 @@ class LoadSuiteBase:
 
         test = cls._test_name(query_num)
         stats = result.stats.get(test)
-        if stats is not None:
-            allure.attach(json.dumps(stats, indent=2), 'Stats', attachment_type=allure.attachment_type.JSON)
-        else:
+        if stats is None:
             stats = {}
         if result.query_out is not None:
             allure.attach(result.query_out, 'Query output', attachment_type=allure.attachment_type.TEXT)
 
-        if result.explain_plan is not None:
+        if result.explain.plan is not None:
             with allure.step('Explain'):
-                _attach_plans(result.explain_plan)
+                _attach_plans(result.explain.plan)
 
-        if result.plans is not None:
-            for i in range(iterations):
-                s = allure.step(f'Iteration {i}')
-                if i in result.time_by_iter:
-                    s.params['duration'] = _duration_text(result.time_by_iter[i])
-                try:
-                    with s:
-                        _attach_plans(result.plans[i])
-                        if i in result.time_by_iter:
-                            allure.dynamic.parameter('duration', _duration_text(result.time_by_iter[i]))
-                        if i in result.errors_by_iter:
-                            pytest.fail(result.errors_by_iter[i])
-                except BaseException:
-                    pass
+        for iter_num in sorted(result.iterations.keys()):
+            iter_res = result.iterations[iter_num]
+            s = allure.step(f'Iteration {iter_num}')
+            if iter_res.time:
+                s.params['duration'] = _duration_text(iter_res.time)
+            try:
+                with s:
+                    _attach_plans(iter_res.plan)
+                    if iter_res.error_message:
+                        pytest.fail(iter_res.error_message)
+            except BaseException:
+                pass
 
         if result.stdout is not None:
             allure.attach(result.stdout, 'Stdout', attachment_type=allure.attachment_type.TEXT)
@@ -222,9 +218,11 @@ class LoadSuiteBase:
             error_message = 'There are fail attemps'
         if os.getenv('NO_KUBER_LOGS') is None and not success:
             cls._attach_logs(start_time=result.start_time, attach_name='kikimr')
+        stats['with_warrnings'] = bool(result.warning_message)
+        stats['with_errors'] = bool(error_message)
+        stats['errors'] = result.get_error_stats()
+        allure.attach(json.dumps(stats, indent=2), 'Stats', attachment_type=allure.attachment_type.JSON)
         if upload:
-            stats['with_warrnings'] = bool(result.warning_message)
-            stats['with_errors'] = bool(error_message)
             ResultsProcessor.upload_results(
                 kind='Load',
                 suite=cls.suite(),