1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
import os
import logging
import shutil
import tempfile
import six
from . import process
from . import runtime
from . import path
yatest_logger = logging.getLogger("ya.test")
def _copy(src, dst, universal_lines=False):
if universal_lines:
with open(dst, "wb") as f:
for line in open(src, "rbU"):
f.write(line)
return
shutil.copy(src, dst)
def canonical_file(path, diff_tool=None, local=False, universal_lines=False, diff_file_name=None, diff_tool_timeout=None):
"""
Create canonical file that can be returned from a test
:param path: path to the file
:param diff_tool: custom diff tool to use for comparison with the canonical one, if None - default will be used
:param local: save file locally, otherwise move to sandbox
:param universal_lines: normalize EOL
:param diff_tool_timeout: timeout for running diff tool
:return: object that can be canonized
"""
abs_path = os.path.abspath(path)
assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
tempdir = tempfile.mkdtemp(prefix="canon_tmp", dir=runtime.build_path())
safe_path = os.path.join(tempdir, os.path.basename(abs_path))
# if the created file is in output_path, we copy it, so that it will be available when the tests finishes
_copy(path, safe_path, universal_lines=universal_lines)
if diff_tool:
if not isinstance(diff_tool, six.string_types):
try: # check if iterable
if not isinstance(diff_tool[0], six.string_types):
raise Exception("Invalid custom diff-tool: not cmd")
except:
raise Exception("Invalid custom diff-tool: not binary path")
return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, local=local, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
def canonical_dir(path, diff_tool=None, diff_file_name=None, diff_tool_timeout=None):
abs_path = os.path.abspath(path)
assert os.path.exists(abs_path), "Canonical path {} does not exist".format(path)
assert os.path.isdir(abs_path), "Path {} is not a directory".format(path)
if diff_file_name and not diff_tool:
raise Exception("diff_file_name can be only be used with diff_tool for canonical_dir")
tempdir = tempfile.mkdtemp()
safe_path = os.path.join(tempdir, os.path.basename(abs_path))
shutil.copytree(abs_path, safe_path)
return runtime._get_ya_plugin_instance().file(safe_path, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
def canonical_execute(
binary, args=None, check_exit_code=True,
shell=False, timeout=None, cwd=None,
env=None, stdin=None, stderr=None, creationflags=0,
file_name=None, save_locally=False, close_fds=False,
diff_tool=None, diff_file_name=None, diff_tool_timeout=None,
):
"""
Shortcut to execute a binary and canonize its stdout
:param binary: absolute path to the binary
:param args: binary arguments
:param check_exit_code: will raise ExecutionError if the command exits with non zero code
:param shell: use shell to run the command
:param timeout: execution timeout
:param cwd: working directory
:param env: command environment
:param stdin: command stdin
:param stderr: command stderr
:param creationflags: command creation flags
:param file_name: output file name. if not specified program name will be used
:param diff_tool: path to custome diff tool
:param diff_file_name: custom diff file name to create when diff is found
:param diff_tool_timeout: timeout for running diff tool
:return: object that can be canonized
"""
if type(binary) == list:
command = binary
else:
command = [binary]
command += _prepare_args(args)
if shell:
command = " ".join(command)
execute_args = locals()
del execute_args["binary"]
del execute_args["args"]
del execute_args["file_name"]
del execute_args["save_locally"]
del execute_args["diff_tool"]
del execute_args["diff_file_name"]
del execute_args["diff_tool_timeout"]
if not file_name and stdin:
file_name = os.path.basename(stdin.name)
return _canonical_execute(process.execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout)
def canonical_py_execute(
script_path, args=None, check_exit_code=True,
shell=False, timeout=None, cwd=None, env=None,
stdin=None, stderr=None, creationflags=0,
file_name=None, save_locally=False, close_fds=False,
diff_tool=None, diff_file_name=None, diff_tool_timeout=None,
):
"""
Shortcut to execute a python script and canonize its stdout
:param script_path: path to the script arcadia relative
:param args: script arguments
:param check_exit_code: will raise ExecutionError if the command exits with non zero code
:param shell: use shell to run the command
:param timeout: execution timeout
:param cwd: working directory
:param env: command environment
:param stdin: command stdin
:param stderr: command stderr
:param creationflags: command creation flags
:param file_name: output file name. if not specified program name will be used
:param diff_tool: path to custome diff tool
:param diff_file_name: custom diff file name to create when diff is found
:param diff_tool_timeout: timeout for running diff tool
:return: object that can be canonized
"""
command = [runtime.source_path(script_path)] + _prepare_args(args)
if shell:
command = " ".join(command)
execute_args = locals()
del execute_args["script_path"]
del execute_args["args"]
del execute_args["file_name"]
del execute_args["save_locally"]
del execute_args["diff_tool"]
del execute_args["diff_file_name"]
del execute_args["diff_tool_timeout"]
return _canonical_execute(process.py_execute, execute_args, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout)
def _prepare_args(args):
if args is None:
args = []
if isinstance(args, six.string_types):
args = map(lambda a: a.strip(), args.split())
return args
def _canonical_execute(excutor, kwargs, file_name, save_locally, diff_tool, diff_file_name, diff_tool_timeout):
res = excutor(**kwargs)
command = kwargs["command"]
file_name = file_name or process.get_command_name(command)
if file_name.endswith(".exe"):
file_name = os.path.splitext(file_name)[0] # don't want to bring windows stuff in file names
out_file_path = path.get_unique_file_path(runtime.output_path(), "{}.out.txt".format(file_name))
err_file_path = path.get_unique_file_path(runtime.output_path(), "{}.err.txt".format(file_name))
try:
os.makedirs(os.path.dirname(out_file_path))
except OSError:
pass
with open(out_file_path, "wb") as out_file:
yatest_logger.debug("Will store file in %s", out_file_path)
out_file.write(res.std_out)
if res.std_err:
with open(err_file_path, "wb") as err_file:
err_file.write(res.std_err)
return canonical_file(out_file_path, local=save_locally, diff_tool=diff_tool, diff_file_name=diff_file_name, diff_tool_timeout=diff_tool_timeout)
|
