1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
#!/usr/bin/env python3
import configparser
import datetime
import os
import time
import ydb
dir = os.path.dirname(__file__)
config = configparser.ConfigParser()
config_file_path = f"{dir}/../../config/ydb_qa_db.ini"
config.read(config_file_path)
DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
def get_test_history(test_names_array, last_n_runs_of_test_amount, build_type):
if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ:
print(
"Error: Env variable CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping"
)
return {}
else:
# Do not set up 'real' variable from gh workflows because it interfere with ydb tests
# So, set up it locally
os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = os.environ[
"CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"
]
results = {}
with ydb.Driver(
endpoint=DATABASE_ENDPOINT,
database=DATABASE_PATH,
credentials=ydb.credentials_from_env_variables(),
) as driver:
driver.wait(timeout=10, fail_fast=True)
session = ydb.retry_operation_sync(
lambda: driver.table_client.session().create()
)
batch_size = 500
start_time = time.time()
for start in range(0, len(test_names_array), batch_size):
test_names_batch = test_names_array[start:start + batch_size]
history_query = f"""
PRAGMA AnsiInForEmptyOrNullableItemsCollections;
DECLARE $test_names AS List<Utf8>;
DECLARE $rn_max AS Int32;
DECLARE $build_type AS Utf8;
$test_names=[{','.join("'{0}'".format(x) for x in test_names_batch)}];
$rn_max = {last_n_runs_of_test_amount};
$build_type = '{build_type}';
$tests=(
SELECT
suite_folder ||'/' || test_name as full_name,test_name,build_type, commit, branch, run_timestamp, status, status_description,
ROW_NUMBER() OVER (PARTITION BY test_name ORDER BY run_timestamp DESC) AS rn
FROM
`test_results/test_runs_column`
where (job_name ='Nightly-run' or job_name like 'Postcommit%') and
build_type = $build_type and
suite_folder ||'/' || test_name in $test_names
and status != 'skipped'
);
select full_name,test_name,build_type, commit, branch, run_timestamp, status, status_description,rn
from $tests
WHERE rn <= $rn_max
ORDER BY test_name, run_timestamp;
"""
query = ydb.ScanQuery(history_query, {})
it = driver.table_client.scan_query(query)
query_result = []
while True:
try:
result = next(it)
query_result = query_result + result.result_set.rows
except StopIteration:
break
for row in query_result:
if not row["full_name"].decode("utf-8") in results:
results[row["full_name"].decode("utf-8")] = {}
results[row["full_name"].decode("utf-8")][row["run_timestamp"]] = {
"status": row["status"],
"commit": row["commit"],
"datetime": datetime.datetime.fromtimestamp(int(row["run_timestamp"] / 1000000)).strftime("%H:%m %B %d %Y"),
"status_description": row["status_description"],
}
end_time = time.time()
print(
f'durations of getting history for {len(test_names_array)} tests :{end_time-start_time} sec')
return results
if __name__ == "__main__":
get_test_history(test_names_array, last_n_runs_of_test_amount, build_type)
|
