from __future__ import print_function
import sys
import os
import re
import subprocess
import signal
import time
import json
import argparse
import errno
# Explicitly enable local imports
# Don't forget to add imported scripts to inputs of the calling command!
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
import process_command_files as pcf
import process_whole_archive_option as pwa
procs = []
build_kekeke = 45
def stringize(s):
return s.encode('utf-8') if isinstance(s, unicode) else s
def run_subprocess(*args, **kwargs):
if 'env' in kwargs:
kwargs['env'] = {stringize(k): stringize(v) for k, v in kwargs['env'].iteritems()}
p = subprocess.Popen(*args, **kwargs)
procs.append(p)
return p
def run_subprocess_with_timeout(timeout, args):
attempts_remaining = 5
delay = 1
p = None
while True:
try:
p = run_subprocess(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate(timeout=timeout)
return p, stdout, stderr
except subprocess.TimeoutExpired as e:
print('timeout running {0}, error {1}, delay {2} seconds'.format(args, str(e), delay), file=sys.stderr)
if p is not None:
try:
p.kill()
p.wait(timeout=1)
except Exception:
pass
attempts_remaining -= 1
if attempts_remaining == 0:
raise
time.sleep(delay)
delay = min(2 * delay, 4)
def terminate_slaves():
for p in procs:
try:
p.terminate()
except Exception:
pass
def sig_term(sig, fr):
terminate_slaves()
sys.exit(sig)
def subst_path(l):
if len(l) > 3:
if l[:3].lower() in ('z:\\', 'z:/'):
return l[2:].replace('\\', '/')
return l
def call_wine_cmd_once(wine, cmd, env, mode):
p = run_subprocess(
wine + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, close_fds=True, shell=False
)
output = find_cmd_out(cmd)
error = None
if output is not None and os.path.exists(output):
try:
os.remove(output)
except OSError as e:
if e.errno != errno.ENOENT:
error = e
except Exception as e:
error = e
if error is not None:
print('Output {} already exists and we have failed to remove it: {}'.format(output, error), file=sys.stderr)
# print >>sys.stderr, cmd, env, wine
stdout_and_stderr, _ = p.communicate()
return_code = p.returncode
if not stdout_and_stderr:
if return_code != 0:
raise Exception('wine did something strange')
return return_code
elif ' : fatal error ' in stdout_and_stderr:
return_code = 1
elif ' : error ' in stdout_and_stderr:
return_code = 2
lines = [x.strip() for x in stdout_and_stderr.split('\n')]
prefixes = [
'Microsoft (R)',
'Copyright (C)',
'Application tried to create a window',
'The graphics driver is missing',
'Could not load wine-gecko',
'wine: configuration in',
'wine: created the configuration directory',
'libpng warning:',
]
suffixes = [
'.c',
'.cxx',
'.cc',
'.cpp',
'.masm',
]
substrs = [
'Creating library Z:',
'err:heap',
'err:menubuilder:',
'err:msvcrt',
'err:ole:',
'err:wincodecs:',
'err:winediag:',
]
def good_line(l):
for x in prefixes:
if l.startswith(x):
return False
for x in suffixes:
if l.endswith(x):
return False
for x in substrs:
if x in l:
return False
return True
def filter_lines():
for l in lines:
if good_line(l):
yield subst_path(l.strip())
stdout_and_stderr = '\n'.join(filter_lines()).strip()
if stdout_and_stderr:
print(stdout_and_stderr, file=sys.stderr)
return return_code
def prepare_vc(fr, to):
for p in os.listdir(fr):
fr_p = os.path.join(fr, p)
to_p = os.path.join(to, p)
if not os.path.exists(to_p):
print('install %s -> %s' % (fr_p, to_p), file=sys.stderr)
os.link(fr_p, to_p)
def run_slave():
args = json.loads(sys.argv[3])
wine = sys.argv[1]
signal.signal(signal.SIGTERM, sig_term)
if args.get('tout', None):
signal.signal(signal.SIGALRM, sig_term)
signal.alarm(args['tout'])
tout = 0.1
while True:
try:
return call_wine_cmd_once([wine], args['cmd'], args['env'], args['mode'])
except Exception as e:
print('%s, will retry in %s' % (str(e), tout), file=sys.stderr)
time.sleep(tout)
tout = min(2 * tout, 4)
def find_cmd_out(args):
for arg in args:
if arg.startswith('/Fo'):
return arg[3:]
if arg.startswith('/OUT:'):
return arg[5:]
def calc_zero_cnt(data):
zero_cnt = 0
for ch in data:
if ch == chr(0):
zero_cnt += 1
return zero_cnt
def is_good_file(p):
if not os.path.isfile(p):
return False
if os.path.getsize(p) < 300:
return False
asm_pattern = re.compile(r'asm(\.\w+)?\.obj$')
if asm_pattern.search(p):
pass
elif p.endswith('.obj'):
with open(p, 'rb') as f:
prefix = f.read(200)
if ord(prefix[0]) != 0:
return False
if ord(prefix[1]) != 0:
return False
if ord(prefix[2]) != 0xFF:
return False
if ord(prefix[3]) != 0xFF:
return False
if calc_zero_cnt(prefix) > 195:
return False
f.seek(-100, os.SEEK_END)
last = f.read(100)
if calc_zero_cnt(last) > 95:
return False
if last[-1] != chr(0):
return False
elif p.endswith('.lib'):
with open(p, 'rb') as f:
if f.read(7) != '!<arch>':
return False
return True
RED = '\x1b[31;1m'
GRAY = '\x1b[30;1m'
RST = '\x1b[0m'
MGT = '\x1b[35m'
YEL = '\x1b[33m'
GRN = '\x1b[32m'
CYA = '\x1b[36m'
def colorize_strings(l):
p = l.find("'")
if p >= 0:
yield l[:p]
l = l[p + 1 :]
p = l.find("'")
if p >= 0:
yield CYA + "'" + subst_path(l[:p]) + "'" + RST
for x in colorize_strings(l[p + 1 :]):
yield x
else:
yield "'" + l
else:
yield l
def colorize_line(l):
lll = l
try:
parts = []
if l.startswith('(compiler file'):
return ''.join(colorize_strings(l))
if l.startswith('/'):
p = l.find('(')
parts.append(GRAY + l[:p] + RST)
l = l[p:]
if l and l.startswith('('):
p = l.find(')')
parts.append(':' + MGT + l[1:p] + RST)
l = l[p + 1 :]
if l:
if l.startswith(' : '):
l = l[1:]
if l.startswith(': error'):
parts.append(': ' + RED + 'error' + RST)
l = l[7:]
elif l.startswith(': warning'):
parts.append(': ' + YEL + 'warning' + RST)
l = l[9:]
elif l.startswith(': note'):
parts.append(': ' + GRN + 'note' + RST)
l = l[6:]
elif l.startswith('fatal error'):
parts.append(RED + 'fatal error' + RST)
l = l[11:]
if l:
parts.extend(colorize_strings(l))
return ''.join(parts)
except Exception:
return lll
def colorize(out):
return '\n'.join(colorize_line(l) for l in out.split('\n'))
def trim_path(path, winepath):
p1, p1_stdout, p1_stderr = run_subprocess_with_timeout(60, [winepath, '-w', path])
win_path = p1_stdout.strip()
if p1.returncode != 0 or not win_path:
# Fall back to only winepath -s
win_path = path
p2, p2_stdout, p2_stderr = run_subprocess_with_timeout(60, [winepath, '-s', win_path])
short_path = p2_stdout.strip()
check_path = short_path
if check_path.startswith(('Z:', 'z:')):
check_path = check_path[2:]
if not check_path[1:].startswith((path[1:4], path[1:4].upper())):
raise Exception(
'Cannot trim path {}; 1st winepath exit code: {}, stdout:\n{}\n stderr:\n{}\n 2nd winepath exit code: {}, stdout:\n{}\n stderr:\n{}'.format(
path, p1.returncode, p1_stdout, p1_stderr, p2.returncode, p2_stdout, p2_stderr
)
)
return short_path
def downsize_path(path, short_names):
flag = ''
if path.startswith('/Fo'):
flag = '/Fo'
path = path[3:]
for full_name, short_name in short_names.items():
if path.startswith(full_name):
path = path.replace(full_name, short_name)
return flag + path
def make_full_path_arg(arg, bld_root, short_root):
if arg[0] != '/' and len(os.path.join(bld_root, arg)) > 250:
return os.path.join(short_root, arg)
return arg
def fix_path(p):
topdirs = ['/%s/' % d for d in os.listdir('/')]
def abs_path_start(path, pos):
if pos < 0:
return False
return pos == 0 or path[pos - 1] == ':'
pp = None
for pr in topdirs:
pp2 = p.find(pr)
if abs_path_start(p, pp2) and (pp is None or pp > pp2):
pp = pp2
if pp is not None:
return p[:pp] + 'Z:' + p[pp:].replace('/', '\\')
if p.startswith('/Fo'):
return '/Fo' + p[3:].replace('/', '\\')
return p
def process_free_args(args, wine, bld_root, mode):
whole_archive_prefix = '/WHOLEARCHIVE:'
short_names = {}
winepath = os.path.join(os.path.dirname(wine), 'winepath')
short_names[bld_root] = trim_path(bld_root, winepath)
# Slow for no benefit.
# arc_root = args.arcadia_root
# short_names[arc_root] = trim_path(arc_root, winepath)
free_args, wa_peers, wa_libs = pwa.get_whole_archive_peers_and_libs(pcf.skip_markers(args))
process_link = lambda x: make_full_path_arg(x, bld_root, short_names[bld_root]) if mode in ('link', 'lib') else x
def process_arg(arg):
with_wa_prefix = arg.startswith(whole_archive_prefix)
prefix = whole_archive_prefix if with_wa_prefix else ''
without_prefix_arg = arg[len(prefix) :]
return prefix + fix_path(process_link(downsize_path(without_prefix_arg, short_names)))
result = []
for arg in free_args:
if pcf.is_cmdfile_arg(arg):
cmd_file_path = pcf.cmdfile_path(arg)
cf_args = pcf.read_from_command_file(cmd_file_path)
with open(cmd_file_path, 'w') as afile:
for cf_arg in cf_args:
afile.write(process_arg(cf_arg) + "\n")
result.append(arg)
else:
result.append(process_arg(arg))
return pwa.ProcessWholeArchiveOption('WINDOWS', wa_peers, wa_libs).construct_cmd(result)
def run_main():
parser = argparse.ArgumentParser()
parser.add_argument('wine', action='store')
parser.add_argument('-v', action='store', dest='version', default='120')
parser.add_argument('-I', action='append', dest='incl_paths')
parser.add_argument('mode', action='store')
parser.add_argument('arcadia_root', action='store')
parser.add_argument('arcadia_build_root', action='store')
parser.add_argument('binary', action='store')
parser.add_argument('free_args', nargs=argparse.REMAINDER)
# By now just unpack. Ideally we should fix path and pack arguments back into command file
args = parser.parse_args()
wine = args.wine
mode = args.mode
binary = args.binary
version = args.version
incl_paths = args.incl_paths
bld_root = args.arcadia_build_root
free_args = args.free_args
wine_dir = os.path.dirname(os.path.dirname(wine))
bin_dir = os.path.dirname(binary)
tc_dir = os.path.dirname(os.path.dirname(os.path.dirname(bin_dir)))
if not incl_paths:
incl_paths = [tc_dir + '/VC/include', tc_dir + '/include']
cmd_out = find_cmd_out(free_args)
env = os.environ.copy()
env.pop('DISPLAY', None)
env['WINEDLLOVERRIDES'] = 'msvcr{}=n'.format(version)
env['WINEDEBUG'] = 'fixme-all'
env['INCLUDE'] = ';'.join(fix_path(p) for p in incl_paths)
env['VSINSTALLDIR'] = fix_path(tc_dir)
env['VCINSTALLDIR'] = fix_path(tc_dir + '/VC')
env['WindowsSdkDir'] = fix_path(tc_dir)
env['LIBPATH'] = fix_path(tc_dir + '/VC/lib/amd64')
env['LIB'] = fix_path(tc_dir + '/VC/lib/amd64')
env['LD_LIBRARY_PATH'] = ':'.join(wine_dir + d for d in ['/lib', '/lib64', '/lib64/wine'])
cmd = [binary] + process_free_args(free_args, wine, bld_root, mode)
for x in ('/NOLOGO', '/nologo', '/FD'):
try:
cmd.remove(x)
except ValueError:
pass
def run_process(sleep, tout):
if sleep:
time.sleep(sleep)
args = {'cmd': cmd, 'env': env, 'mode': mode, 'tout': tout}
slave_cmd = [sys.executable, sys.argv[0], wine, 'slave', json.dumps(args)]
p = run_subprocess(slave_cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=False)
out, _ = p.communicate()
return p.wait(), out
def print_err_log(log):
if not log:
return
if mode == 'cxx':
log = colorize(log)
print(log, file=sys.stderr)
tout = 200
while True:
rc, out = run_process(0, tout)
if rc in (-signal.SIGALRM, signal.SIGALRM):
print_err_log(out)
print('##append_tag##time out', file=sys.stderr)
elif out and ' stack overflow ' in out:
print('##append_tag##stack overflow', file=sys.stderr)
elif out and 'recvmsg: Connection reset by peer' in out:
print('##append_tag##wine gone', file=sys.stderr)
elif out and 'D8037' in out:
print('##append_tag##repair wine', file=sys.stderr)
try:
os.unlink(os.path.join(os.environ['WINEPREFIX'], '.update-timestamp'))
except Exception as e:
print(e, file=sys.stderr)
else:
print_err_log(out)
# non-zero return code - bad, return it immediately
if rc:
print('##win_cmd##' + ' '.join(cmd), file=sys.stderr)
print('##args##' + ' '.join(free_args), file=sys.stderr)
return rc
# check for output existence(if we expect it!) and real length
if cmd_out:
if is_good_file(cmd_out):
return 0
else:
# retry!
print('##append_tag##no output', file=sys.stderr)
else:
return 0
tout *= 3
def main():
prefix_suffix = os.environ.pop('WINEPREFIX_SUFFIX', None)
if prefix_suffix is not None:
prefix = os.environ.pop('WINEPREFIX', None)
if prefix is not None:
os.environ['WINEPREFIX'] = os.path.join(prefix, prefix_suffix)
# just in case
signal.alarm(2000)
if sys.argv[2] == 'slave':
func = run_slave
else:
func = run_main
try:
try:
sys.exit(func())
finally:
terminate_slaves()
except KeyboardInterrupt:
sys.exit(4)
except Exception as e:
print(str(e), file=sys.stderr)
sys.exit(3)
if __name__ == '__main__':
main()