diff options
author | dvshkurko <dvshkurko@yandex-team.ru> | 2022-02-10 16:45:52 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:52 +0300 |
commit | c768a99151e47c3a4bb7b92c514d256abd301c4d (patch) | |
tree | 1a2c5ffcf89eb53ecd79dbc9bc0a195c27404d0c /build | |
parent | 321ee9bce31ec6e238be26dbcbe539cffa2c3309 (diff) | |
download | ydb-c768a99151e47c3a4bb7b92c514d256abd301c4d.tar.gz |
Restoring authorship annotation for <dvshkurko@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'build')
-rw-r--r-- | build/docs/empty/ya.make | 10 | ||||
-rw-r--r-- | build/plugins/java.py | 18 | ||||
-rw-r--r-- | build/scripts/c_templates/svn_interface.c | 314 | ||||
-rw-r--r-- | build/scripts/c_templates/svnversion.h | 78 | ||||
-rw-r--r-- | build/scripts/c_templates/ya.make | 10 | ||||
-rw-r--r-- | build/scripts/compile_java.py | 18 | ||||
-rwxr-xr-x | build/scripts/fetch_from.py | 62 | ||||
-rw-r--r-- | build/scripts/fetch_from_external.py | 2 | ||||
-rw-r--r-- | build/scripts/fetch_from_mds.py | 2 | ||||
-rwxr-xr-x | build/scripts/fetch_from_sandbox.py | 158 | ||||
-rw-r--r-- | build/scripts/go_tool.py | 12 | ||||
-rw-r--r-- | build/scripts/link_exe.py | 36 | ||||
-rw-r--r-- | build/scripts/run_msvc_wine.py | 6 | ||||
-rw-r--r-- | build/scripts/vcs_info.py | 500 | ||||
-rw-r--r-- | build/scripts/ya.make | 4 | ||||
-rw-r--r-- | build/ya.conf.json | 26 | ||||
-rw-r--r-- | build/ya.make | 2 | ||||
-rw-r--r-- | build/ymake.core.conf | 142 | ||||
-rwxr-xr-x | build/ymake_conf.py | 142 |
19 files changed, 771 insertions, 771 deletions
diff --git a/build/docs/empty/ya.make b/build/docs/empty/ya.make index 694964e428..3540411405 100644 --- a/build/docs/empty/ya.make +++ b/build/docs/empty/ya.make @@ -1,5 +1,5 @@ -UNION() - -OWNER(g:ymake) - -END() +UNION() + +OWNER(g:ymake) + +END() diff --git a/build/plugins/java.py b/build/plugins/java.py index c40e184295..16fc126734 100644 --- a/build/plugins/java.py +++ b/build/plugins/java.py @@ -180,15 +180,15 @@ def onjava_module(unit, *args): ymake.report_configure_error('{}: JDK export supported only for JAVA_PROGRAM module type'.format(unit.path())) data['WITH_JDK'] = extract_macro_calls(unit, 'WITH_JDK_VALUE', args_delim) - if not data['EXTERNAL_JAR']: - has_processor = extract_macro_calls(unit, 'GENERATE_VCS_JAVA_INFO_NODEP', args_delim) - data['EMBED_VCS'] = [[str(has_processor and has_processor[0] and has_processor[0][0])]] - # FORCE_VCS_INFO_UPDATE is responsible for setting special value of VCS_INFO_DISABLE_CACHE__NO_UID__ - macro_val = extract_macro_calls(unit, 'FORCE_VCS_INFO_UPDATE', args_delim) - macro_str = macro_val[0][0] if macro_val and macro_val[0] and macro_val[0][0] else '' - if macro_str and macro_str == 'yes': - data['VCS_INFO_DISABLE_CACHE__NO_UID__'] = macro_val - + if not data['EXTERNAL_JAR']: + has_processor = extract_macro_calls(unit, 'GENERATE_VCS_JAVA_INFO_NODEP', args_delim) + data['EMBED_VCS'] = [[str(has_processor and has_processor[0] and has_processor[0][0])]] + # FORCE_VCS_INFO_UPDATE is responsible for setting special value of VCS_INFO_DISABLE_CACHE__NO_UID__ + macro_val = extract_macro_calls(unit, 'FORCE_VCS_INFO_UPDATE', args_delim) + macro_str = macro_val[0][0] if macro_val and macro_val[0] and macro_val[0][0] else '' + if macro_str and macro_str == 'yes': + data['VCS_INFO_DISABLE_CACHE__NO_UID__'] = macro_val + for java_srcs_args in data['JAVA_SRCS']: external = None diff --git a/build/scripts/c_templates/svn_interface.c b/build/scripts/c_templates/svn_interface.c index 45cb2b09b5..d8bdd1a70d 100644 --- a/build/scripts/c_templates/svn_interface.c +++ b/build/scripts/c_templates/svn_interface.c @@ -1,79 +1,79 @@ -// Used indirectly. See build/scripts/vcs_info.py -// ya-bin dump vcs-info > vcs.json +// Used indirectly. See build/scripts/vcs_info.py +// ya-bin dump vcs-info > vcs.json // python build/scripts/vcs_info.py vcs.json out.c build/scripts/c_templates/svn_interface.c <any_prefix>library/cpp/svnversion<any_suffix> - - -#include "build/scripts/c_templates/svnversion.h" - -#define STR1(x) #x -#define STR2(x) STR1(x) - -const char* GetProgramSvnVersion() { -#if defined(REVISION) -// for package systems generating from svn export but providing REVISION macro -#define STRREVISION STR2(REVISION) -#define REVISIONINFO "r" STRREVISION -#if defined(PROGRAM_VERSION) - return PROGRAM_VERSION "\n\n" REVISIONINFO; -#else - return REVISIONINFO " "__DATE__ - " "__TIME__; -#endif -#elif defined(PROGRAM_VERSION) - return PROGRAM_VERSION; -#else - return "No program version found"; -#endif -} - -const char* GetArcadiaSourcePath() { -#if defined(ARCADIA_SOURCE_PATH) - return ARCADIA_SOURCE_PATH; -#else - return ""; -#endif -} - -const char* GetArcadiaSourceUrl() { -#if defined(ARCADIA_SOURCE_URL) - return ARCADIA_SOURCE_URL; -#else - return ""; -#endif -} - -int GetArcadiaLastChangeNum() { -#if defined(ARCADIA_SOURCE_LAST_CHANGE) - return ARCADIA_SOURCE_LAST_CHANGE; -#else - return 0; -#endif -} - -const char* GetArcadiaLastChange() { -#if defined(ARCADIA_SOURCE_LAST_CHANGE) - return STR2(ARCADIA_SOURCE_LAST_CHANGE); -#else - return ""; -#endif -} - -const char* GetArcadiaLastAuthor() { -#if defined(ARCADIA_SOURCE_LAST_AUTHOR) - return ARCADIA_SOURCE_LAST_AUTHOR; -#else - return ""; -#endif -} - -int GetProgramSvnRevision() { -#if defined(ARCADIA_SOURCE_REVISION) - return ARCADIA_SOURCE_REVISION; -#else - return 0; -#endif -} - + + +#include "build/scripts/c_templates/svnversion.h" + +#define STR1(x) #x +#define STR2(x) STR1(x) + +const char* GetProgramSvnVersion() { +#if defined(REVISION) +// for package systems generating from svn export but providing REVISION macro +#define STRREVISION STR2(REVISION) +#define REVISIONINFO "r" STRREVISION +#if defined(PROGRAM_VERSION) + return PROGRAM_VERSION "\n\n" REVISIONINFO; +#else + return REVISIONINFO " "__DATE__ + " "__TIME__; +#endif +#elif defined(PROGRAM_VERSION) + return PROGRAM_VERSION; +#else + return "No program version found"; +#endif +} + +const char* GetArcadiaSourcePath() { +#if defined(ARCADIA_SOURCE_PATH) + return ARCADIA_SOURCE_PATH; +#else + return ""; +#endif +} + +const char* GetArcadiaSourceUrl() { +#if defined(ARCADIA_SOURCE_URL) + return ARCADIA_SOURCE_URL; +#else + return ""; +#endif +} + +int GetArcadiaLastChangeNum() { +#if defined(ARCADIA_SOURCE_LAST_CHANGE) + return ARCADIA_SOURCE_LAST_CHANGE; +#else + return 0; +#endif +} + +const char* GetArcadiaLastChange() { +#if defined(ARCADIA_SOURCE_LAST_CHANGE) + return STR2(ARCADIA_SOURCE_LAST_CHANGE); +#else + return ""; +#endif +} + +const char* GetArcadiaLastAuthor() { +#if defined(ARCADIA_SOURCE_LAST_AUTHOR) + return ARCADIA_SOURCE_LAST_AUTHOR; +#else + return ""; +#endif +} + +int GetProgramSvnRevision() { +#if defined(ARCADIA_SOURCE_REVISION) + return ARCADIA_SOURCE_REVISION; +#else + return 0; +#endif +} + const char* GetVCSDirty() { #if defined(DIRTY) @@ -83,33 +83,33 @@ const char* GetVCSDirty() #endif } -const char* GetProgramHash() { -#if defined(ARCADIA_SOURCE_HG_HASH) - return ARCADIA_SOURCE_HG_HASH; -#else - return ""; -#endif -} - -const char* GetProgramCommitId() { -#if defined(ARCADIA_SOURCE_REVISION) - if (ARCADIA_SOURCE_REVISION <= 0) { - return GetProgramHash(); - } - return STR2(ARCADIA_SOURCE_REVISION); -#else - return GetProgramHash(); -#endif -} - -const char* GetProgramScmData() { -#if defined(SCM_DATA) - return SCM_DATA; -#else - return ""; -#endif -} - +const char* GetProgramHash() { +#if defined(ARCADIA_SOURCE_HG_HASH) + return ARCADIA_SOURCE_HG_HASH; +#else + return ""; +#endif +} + +const char* GetProgramCommitId() { +#if defined(ARCADIA_SOURCE_REVISION) + if (ARCADIA_SOURCE_REVISION <= 0) { + return GetProgramHash(); + } + return STR2(ARCADIA_SOURCE_REVISION); +#else + return GetProgramHash(); +#endif +} + +const char* GetProgramScmData() { +#if defined(SCM_DATA) + return SCM_DATA; +#else + return ""; +#endif +} + const char* GetProgramShortVersionData() { #if defined(SVN_REVISION) && defined(SVN_TIME) return STR2(SVN_REVISION) " (" SVN_TIME ")"; @@ -118,30 +118,30 @@ const char* GetProgramShortVersionData() { #endif } -const char* GetProgramBuildUser() { -#if defined(BUILD_USER) - return BUILD_USER; -#else - return ""; -#endif -} - -const char* GetProgramBuildHost() { -#if defined(BUILD_HOST) - return BUILD_HOST; -#else - return ""; -#endif -} - -const char* GetProgramBuildDate() { -#if defined(BUILD_DATE) - return BUILD_DATE; -#else - return ""; -#endif -} - +const char* GetProgramBuildUser() { +#if defined(BUILD_USER) + return BUILD_USER; +#else + return ""; +#endif +} + +const char* GetProgramBuildHost() { +#if defined(BUILD_HOST) + return BUILD_HOST; +#else + return ""; +#endif +} + +const char* GetProgramBuildDate() { +#if defined(BUILD_DATE) + return BUILD_DATE; +#else + return ""; +#endif +} + int GetProgramBuildTimestamp() { #if defined(BUILD_TIMESTAMP) return BUILD_TIMESTAMP; @@ -151,34 +151,34 @@ int GetProgramBuildTimestamp() { } -const char* GetVCS() { -#if defined(VCS) - return VCS; -#else - return ""; -#endif -} - -const char* GetBranch() { -#if defined(BRANCH) - return BRANCH; -#else - return ""; -#endif -} - -int GetArcadiaPatchNumber() { -#if defined(ARCADIA_PATCH_NUMBER) - return ARCADIA_PATCH_NUMBER; -#else - return 42; -#endif -} - -const char* GetTag() { -#if defined(ARCADIA_TAG) - return ARCADIA_TAG; -#else - return ""; -#endif -} +const char* GetVCS() { +#if defined(VCS) + return VCS; +#else + return ""; +#endif +} + +const char* GetBranch() { +#if defined(BRANCH) + return BRANCH; +#else + return ""; +#endif +} + +int GetArcadiaPatchNumber() { +#if defined(ARCADIA_PATCH_NUMBER) + return ARCADIA_PATCH_NUMBER; +#else + return 42; +#endif +} + +const char* GetTag() { +#if defined(ARCADIA_TAG) + return ARCADIA_TAG; +#else + return ""; +#endif +} diff --git a/build/scripts/c_templates/svnversion.h b/build/scripts/c_templates/svnversion.h index 3cc1adab35..7aece931aa 100644 --- a/build/scripts/c_templates/svnversion.h +++ b/build/scripts/c_templates/svnversion.h @@ -1,42 +1,42 @@ -#pragma once - -// Permit compilation with NO_UTIL(): -// util/system/compiler.h -#if !defined(Y_HIDDEN) -#if defined(__GNUC__) -#define Y_HIDDEN __attribute__((visibility("hidden"))) -#else -#define Y_HIDDEN -#endif -#endif - -#if defined(__cplusplus) -extern "C" { -#endif -const char* GetProgramSvnVersion() Y_HIDDEN; // verbose multiline message -void PrintProgramSvnVersion() Y_HIDDEN; -const char* GetArcadiaSourcePath() Y_HIDDEN; // "/home/myltsev/arcadia" -const char* GetArcadiaSourceUrl() Y_HIDDEN; // "svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia" -const char* GetArcadiaLastChange() Y_HIDDEN; // "2902074" -int GetArcadiaLastChangeNum() Y_HIDDEN; // 2902074 -const char* GetArcadiaLastAuthor() Y_HIDDEN; // "dieash" -int GetProgramSvnRevision() Y_HIDDEN; // 2902074 +#pragma once + +// Permit compilation with NO_UTIL(): +// util/system/compiler.h +#if !defined(Y_HIDDEN) +#if defined(__GNUC__) +#define Y_HIDDEN __attribute__((visibility("hidden"))) +#else +#define Y_HIDDEN +#endif +#endif + +#if defined(__cplusplus) +extern "C" { +#endif +const char* GetProgramSvnVersion() Y_HIDDEN; // verbose multiline message +void PrintProgramSvnVersion() Y_HIDDEN; +const char* GetArcadiaSourcePath() Y_HIDDEN; // "/home/myltsev/arcadia" +const char* GetArcadiaSourceUrl() Y_HIDDEN; // "svn+ssh://arcadia.yandex.ru/arc/trunk/arcadia" +const char* GetArcadiaLastChange() Y_HIDDEN; // "2902074" +int GetArcadiaLastChangeNum() Y_HIDDEN; // 2902074 +const char* GetArcadiaLastAuthor() Y_HIDDEN; // "dieash" +int GetProgramSvnRevision() Y_HIDDEN; // 2902074 const char* GetVCSDirty() Y_HIDDEN; -const char* GetProgramHash() Y_HIDDEN; -const char* GetProgramCommitId() Y_HIDDEN; -void PrintSvnVersionAndExit(int argc, char* argv[]) Y_HIDDEN; -void PrintSvnVersionAndExitEx(int argc, char* argv[], const char* opts) Y_HIDDEN; -void PrintSvnVersionAndExit0() Y_HIDDEN; -const char* GetProgramScmData() Y_HIDDEN; // verbose multiline message +const char* GetProgramHash() Y_HIDDEN; +const char* GetProgramCommitId() Y_HIDDEN; +void PrintSvnVersionAndExit(int argc, char* argv[]) Y_HIDDEN; +void PrintSvnVersionAndExitEx(int argc, char* argv[], const char* opts) Y_HIDDEN; +void PrintSvnVersionAndExit0() Y_HIDDEN; +const char* GetProgramScmData() Y_HIDDEN; // verbose multiline message const char* GetProgramShortVersionData() Y_HIDDEN; -const char* GetProgramBuildUser() Y_HIDDEN; -const char* GetProgramBuildHost() Y_HIDDEN; -const char* GetProgramBuildDate() Y_HIDDEN; +const char* GetProgramBuildUser() Y_HIDDEN; +const char* GetProgramBuildHost() Y_HIDDEN; +const char* GetProgramBuildDate() Y_HIDDEN; int GetProgramBuildTimestamp() Y_HIDDEN; -const char* GetVCS() Y_HIDDEN; -const char* GetBranch() Y_HIDDEN; -const char* GetTag() Y_HIDDEN; -int GetArcadiaPatchNumber() Y_HIDDEN; -#if defined(__cplusplus) -} -#endif +const char* GetVCS() Y_HIDDEN; +const char* GetBranch() Y_HIDDEN; +const char* GetTag() Y_HIDDEN; +int GetArcadiaPatchNumber() Y_HIDDEN; +#if defined(__cplusplus) +} +#endif diff --git a/build/scripts/c_templates/ya.make b/build/scripts/c_templates/ya.make index 306077cc96..b395e11021 100644 --- a/build/scripts/c_templates/ya.make +++ b/build/scripts/c_templates/ya.make @@ -1,7 +1,7 @@ OWNER(g:ymake) -LIBRARY(dummy-vcs) -NO_PLATFORM() - -SRCS(svn_interface.c) -END() +LIBRARY(dummy-vcs) +NO_PLATFORM() + +SRCS(svn_interface.c) +END() diff --git a/build/scripts/compile_java.py b/build/scripts/compile_java.py index ad3164355d..e95869e853 100644 --- a/build/scripts/compile_java.py +++ b/build/scripts/compile_java.py @@ -12,7 +12,7 @@ def parse_args(args): parser = optparse.OptionParser() parser.add_option('--javac-bin') parser.add_option('--jar-bin') - parser.add_option('--vcs-mf') + parser.add_option('--vcs-mf') parser.add_option('--package-prefix') parser.add_option('--jar-output') parser.add_option('--srcs-jar-output') @@ -76,10 +76,10 @@ def main(): with zipfile.ZipFile(s) as zf: zf.extractall(classes_dir) - if opts.vcs_mf: - sp.check_call([opts.jar_bin, 'cfm', opts.jar_output, opts.vcs_mf, os.curdir], cwd=classes_dir) - else: - sp.check_call([opts.jar_bin, 'cfM', opts.jar_output, os.curdir], cwd=classes_dir) + if opts.vcs_mf: + sp.check_call([opts.jar_bin, 'cfm', opts.jar_output, opts.vcs_mf, os.curdir], cwd=classes_dir) + else: + sp.check_call([opts.jar_bin, 'cfM', opts.jar_output, os.curdir], cwd=classes_dir) if opts.srcs_jar_output: for s in jsrcs: @@ -92,10 +92,10 @@ def main(): shutil.copyfile(s, d) - if opts.vcs_mf: - sp.check_call([opts.jar_bin, 'cfm', opts.srcs_jar_output, opts.vcs_mf, os.curdir], cwd=sources_dir) - else: - sp.check_call([opts.jar_bin, 'cfM', opts.srcs_jar_output, os.curdir], cwd=sources_dir) + if opts.vcs_mf: + sp.check_call([opts.jar_bin, 'cfm', opts.srcs_jar_output, opts.vcs_mf, os.curdir], cwd=sources_dir) + else: + sp.check_call([opts.jar_bin, 'cfM', opts.srcs_jar_output, os.curdir], cwd=sources_dir) if __name__ == '__main__': diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py index 96f3acd589..db4fea50bf 100755 --- a/build/scripts/fetch_from.py +++ b/build/scripts/fetch_from.py @@ -4,7 +4,7 @@ import hashlib import json import logging import os -import platform +import platform import random import shutil import socket @@ -27,7 +27,7 @@ def add_common_arguments(parser): parser.add_argument('--untar-to') parser.add_argument('--rename', action='append', default=[], metavar='FILE', help='rename FILE to the corresponding output') parser.add_argument('--executable', action='store_true', help='make outputs executable') - parser.add_argument('--log-path') + parser.add_argument('--log-path') parser.add_argument('-v', '--verbose', action='store_true', default=os.environ.get('YA_VERBOSE_FETCHER'), help='increase stderr verbosity') parser.add_argument('outputs', nargs='*', default=[]) @@ -90,36 +90,36 @@ class OutputNotExistError(Exception): pass -def setup_logging(args, base_name): - def makedirs(path): - try: - os.makedirs(path) - except OSError: - pass - - if args.log_path: - log_file_name = args.log_path - else: - log_file_name = base_name + ".log" - - args.abs_log_path = os.path.abspath(log_file_name) - makedirs(os.path.dirname(args.abs_log_path)) - logging.basicConfig(filename=args.abs_log_path, level=logging.DEBUG) +def setup_logging(args, base_name): + def makedirs(path): + try: + os.makedirs(path) + except OSError: + pass + + if args.log_path: + log_file_name = args.log_path + else: + log_file_name = base_name + ".log" + + args.abs_log_path = os.path.abspath(log_file_name) + makedirs(os.path.dirname(args.abs_log_path)) + logging.basicConfig(filename=args.abs_log_path, level=logging.DEBUG) if args.verbose: logging.getLogger().addHandler(logging.StreamHandler(sys.stderr)) - - + + def is_temporary(e): - def is_broken(e): - return isinstance(e, urllib2.HTTPError) and e.code in (410, 404) + def is_broken(e): + return isinstance(e, urllib2.HTTPError) and e.code in (410, 404) if is_broken(e): return False - + if isinstance(e, (BadChecksumFetchError, IncompleteFetchError, urllib2.URLError, socket.error)): return True - + import error return error.is_temporary_error(e) @@ -293,16 +293,16 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1= def chmod(filename, mode): - if platform.system().lower() == 'windows': - # https://docs.microsoft.com/en-us/windows/win32/fileio/hard-links-and-junctions: - # hard to reset read-only attribute for removal if there are multiple hardlinks - return + if platform.system().lower() == 'windows': + # https://docs.microsoft.com/en-us/windows/win32/fileio/hard-links-and-junctions: + # hard to reset read-only attribute for removal if there are multiple hardlinks + return stat = os.stat(filename) if stat.st_mode & 0o777 != mode: try: os.chmod(filename, mode) except OSError: - import pwd + import pwd sys.stderr.write("{} st_mode: {} pwuid: {}\n".format(filename, stat.st_mode, pwd.getpwuid(os.stat(filename).st_uid))) raise @@ -356,9 +356,9 @@ def process(fetched_file, file_name, args, remove=True): logging.info('Renaming %s to %s', src, dst) if os.path.exists(dst): raise ResourceUnpackingError("Target file already exists ({} -> {})".format(src, dst)) - if remove: - rename_or_copy_and_remove(src, dst) - else: + if remove: + rename_or_copy_and_remove(src, dst) + else: hardlink_or_copy(src, dst) for path in args.outputs: diff --git a/build/scripts/fetch_from_external.py b/build/scripts/fetch_from_external.py index c7202d2ed2..d4ed6f4221 100644 --- a/build/scripts/fetch_from_external.py +++ b/build/scripts/fetch_from_external.py @@ -18,7 +18,7 @@ def parse_args(): def main(args): external_file = args.external_file.rstrip('.external') if os.path.isfile(args.resource_file): - fetch_from.process(args.resource_file, os.path.basename(args.resource_file), args, False) + fetch_from.process(args.resource_file, os.path.basename(args.resource_file), args, False) return error = None diff --git a/build/scripts/fetch_from_mds.py b/build/scripts/fetch_from_mds.py index 7ab993fc31..5e4e656394 100644 --- a/build/scripts/fetch_from_mds.py +++ b/build/scripts/fetch_from_mds.py @@ -37,7 +37,7 @@ def main(args): if __name__ == '__main__': args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) + fetch_from.setup_logging(args, os.path.basename(__file__)) try: main(args) diff --git a/build/scripts/fetch_from_sandbox.py b/build/scripts/fetch_from_sandbox.py index 5408ca05af..a99542e174 100755 --- a/build/scripts/fetch_from_sandbox.py +++ b/build/scripts/fetch_from_sandbox.py @@ -15,7 +15,7 @@ import fetch_from ORIGIN_SUFFIX = '?origin=fetch-from-sandbox' MDS_PREFIX = 'http://storage-int.mds.yandex.net/get-sandbox/' -TEMPORARY_ERROR_CODES = (429, 500, 503, 504) +TEMPORARY_ERROR_CODES = (429, 500, 503, 504) def parse_args(): @@ -23,7 +23,7 @@ def parse_args(): fetch_from.add_common_arguments(parser) parser.add_argument('--resource-id', type=int, required=True) parser.add_argument('--custom-fetcher') - parser.add_argument('--resource-file') + parser.add_argument('--resource-file') return parser.parse_args() @@ -35,33 +35,33 @@ class UnsupportedProtocolException(Exception): pass -def _sky_path(): - return "/usr/local/bin/sky" +def _sky_path(): + return "/usr/local/bin/sky" -def _is_skynet_avaliable(): - if not os.path.exists(_sky_path()): - return False - try: - subprocess.check_output([_sky_path(), "--version"]) - return True - except subprocess.CalledProcessError: - return False - except OSError: - return False - - -def download_by_skynet(resource_info, file_name): +def _is_skynet_avaliable(): + if not os.path.exists(_sky_path()): + return False + try: + subprocess.check_output([_sky_path(), "--version"]) + return True + except subprocess.CalledProcessError: + return False + except OSError: + return False + + +def download_by_skynet(resource_info, file_name): def sky_get(skynet_id, target_dir, timeout=None): cmd_args = [_sky_path(), 'get', "-N", "Backbone", "--user", "--wait", "--dir", target_dir, skynet_id] if timeout is not None: cmd_args += ["--timeout", str(timeout)] - logging.info('Call skynet with args: %s', cmd_args) + logging.info('Call skynet with args: %s', cmd_args) stdout = subprocess.check_output(cmd_args).strip() logging.debug('Skynet call with args %s is finished, result is %s', cmd_args, stdout) return stdout - if not _is_skynet_avaliable(): + if not _is_skynet_avaliable(): raise UnsupportedProtocolException("Skynet is not available") skynet_id = resource_info.get("skynet_id") @@ -94,7 +94,7 @@ def _urlopen(url, data=None, headers=None): logging.warning('failed to fetch URL %s with HTTP code %d: %s', url, e.code, e) retry_after = int(e.headers.get('Retry-After', str(retry_after))) - if e.code not in TEMPORARY_ERROR_CODES: + if e.code not in TEMPORARY_ERROR_CODES: raise except Exception as e: @@ -141,9 +141,9 @@ def fetch(resource_id, custom_fetcher): ) raise - if resource_info.get('state', 'DELETED') != 'READY': - raise ResourceInfoError("Resource {} is not READY".format(resource_id)) - + if resource_info.get('state', 'DELETED') != 'READY': + raise ResourceInfoError("Resource {} is not READY".format(resource_id)) + logging.info('Resource %s info %s', str(resource_id), json.dumps(resource_info)) resource_file_name = os.path.basename(resource_info["file_name"]) @@ -159,15 +159,15 @@ def fetch(resource_id, custom_fetcher): random.shuffle(storage_links) return storage_links - skynet = _is_skynet_avaliable() - - if not skynet: - logging.info("Skynet is not available, will try other protocols") - + skynet = _is_skynet_avaliable() + + if not skynet: + logging.info("Skynet is not available, will try other protocols") + def iter_tries(): - if skynet: - yield lambda: download_by_skynet(resource_info, resource_file_name) - + if skynet: + yield lambda: download_by_skynet(resource_info, resource_file_name) + if custom_fetcher: yield lambda: fetch_via_script(custom_fetcher, resource_id) @@ -191,78 +191,78 @@ def fetch(resource_id, custom_fetcher): try: fetched_file = action() break - except UnsupportedProtocolException: - pass - except subprocess.CalledProcessError as e: + except UnsupportedProtocolException: + pass + except subprocess.CalledProcessError as e: logging.warning('failed to fetch resource %s with subprocess: %s', resource_id, e) - time.sleep(i) - except urllib2.HTTPError as e: + time.sleep(i) + except urllib2.HTTPError as e: logging.warning('failed to fetch resource %s with HTTP code %d: %s', resource_id, e.code, e) - if e.code not in TEMPORARY_ERROR_CODES: - exc_info = exc_info or sys.exc_info() - time.sleep(i) + if e.code not in TEMPORARY_ERROR_CODES: + exc_info = exc_info or sys.exc_info() + time.sleep(i) except Exception as e: logging.exception(e) exc_info = exc_info or sys.exc_info() time.sleep(i) else: - if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] - else: - raise Exception("No available protocol and/or server to fetch resource") + if exc_info: + raise exc_info[0], exc_info[1], exc_info[2] + else: + raise Exception("No available protocol and/or server to fetch resource") return fetched_file, resource_info['file_name'] -def _get_resource_info_from_file(resource_file): - if resource_file is None or not os.path.exists(resource_file): - return None - - RESOURCE_INFO_JSON = "resource_info.json" - RESOURCE_CONTENT_FILE_NAME = "resource" - - resource_dir, resource_file = os.path.split(resource_file) - if resource_file != RESOURCE_CONTENT_FILE_NAME: - return None - - resource_json = os.path.join(resource_dir, RESOURCE_INFO_JSON) - if not os.path.isfile(resource_json): - return None - - try: - with open(resource_json, 'r') as j: - resource_info = json.load(j) - resource_info['file_name'] # check consistency - return resource_info - except: - logging.debug('Invalid %s in %s', RESOURCE_INFO_JSON, resource_dir) - - return None - - +def _get_resource_info_from_file(resource_file): + if resource_file is None or not os.path.exists(resource_file): + return None + + RESOURCE_INFO_JSON = "resource_info.json" + RESOURCE_CONTENT_FILE_NAME = "resource" + + resource_dir, resource_file = os.path.split(resource_file) + if resource_file != RESOURCE_CONTENT_FILE_NAME: + return None + + resource_json = os.path.join(resource_dir, RESOURCE_INFO_JSON) + if not os.path.isfile(resource_json): + return None + + try: + with open(resource_json, 'r') as j: + resource_info = json.load(j) + resource_info['file_name'] # check consistency + return resource_info + except: + logging.debug('Invalid %s in %s', RESOURCE_INFO_JSON, resource_dir) + + return None + + def main(args): custom_fetcher = os.environ.get('YA_CUSTOM_FETCHER') - resource_info = _get_resource_info_from_file(args.resource_file) - if resource_info: - fetched_file = args.resource_file - file_name = resource_info['file_name'] - else: - # This code should be merged to ya and removed. - fetched_file, file_name = fetch(args.resource_id, custom_fetcher) + resource_info = _get_resource_info_from_file(args.resource_file) + if resource_info: + fetched_file = args.resource_file + file_name = resource_info['file_name'] + else: + # This code should be merged to ya and removed. + fetched_file, file_name = fetch(args.resource_id, custom_fetcher) - fetch_from.process(fetched_file, file_name, args, remove=not custom_fetcher and not resource_info) + fetch_from.process(fetched_file, file_name, args, remove=not custom_fetcher and not resource_info) if __name__ == '__main__': args = parse_args() - fetch_from.setup_logging(args, os.path.basename(__file__)) + fetch_from.setup_logging(args, os.path.basename(__file__)) try: main(args) except Exception as e: logging.exception(e) - print >>sys.stderr, open(args.abs_log_path).read() + print >>sys.stderr, open(args.abs_log_path).read() sys.stderr.flush() import error diff --git a/build/scripts/go_tool.py b/build/scripts/go_tool.py index a25a871a94..c1e98b20c0 100644 --- a/build/scripts/go_tool.py +++ b/build/scripts/go_tool.py @@ -474,12 +474,12 @@ def do_link_exe(args): compile_args.output = os.path.join(args.output_root, 'main.a') compile_args.real_import_path = compile_args.import_path compile_args.import_path = 'main' - - if args.vcs and os.path.isfile(compile_args.vcs): - build_info = os.path.join('library', 'go', 'core', 'buildinfo') + + if args.vcs and os.path.isfile(compile_args.vcs): + build_info = os.path.join('library', 'go', 'core', 'buildinfo') if any([x.startswith(build_info) for x in compile_args.peers]): - compile_args.go_srcs.append(compile_args.vcs) - + compile_args.go_srcs.append(compile_args.vcs) + do_link_lib(compile_args) cmd = [args.go_link, '-o', args.output] import_config_name = create_import_config(args.peers + args.non_local_peers, False, args.import_map, args.module_map) @@ -814,7 +814,7 @@ if __name__ == '__main__': parser.add_argument('++asm-flags', nargs='*') parser.add_argument('++compile-flags', nargs='*') parser.add_argument('++link-flags', nargs='*') - parser.add_argument('++vcs', nargs='?', default=None) + parser.add_argument('++vcs', nargs='?', default=None) parser.add_argument('++vet', nargs='?', const=True, default=False) parser.add_argument('++vet-flags', nargs='*', default=None) parser.add_argument('++vet-info-ext', default=vet_info_ext) diff --git a/build/scripts/link_exe.py b/build/scripts/link_exe.py index d89528b6fa..f469e3b442 100644 --- a/build/scripts/link_exe.py +++ b/build/scripts/link_exe.py @@ -1,6 +1,6 @@ import sys import subprocess -import optparse +import optparse from process_whole_archive_option import ProcessWholeArchiveOption @@ -15,19 +15,19 @@ def get_leaks_suppressions(cmd): return supp, newcmd -musl_libs = '-lc', '-lcrypt', '-ldl', '-lm', '-lpthread', '-lrt', '-lutil' - - -def fix_cmd(musl, c): +musl_libs = '-lc', '-lcrypt', '-ldl', '-lm', '-lpthread', '-lrt', '-lutil' + + +def fix_cmd(musl, c): return [i for i in c if (not musl or i not in musl_libs) and not i.endswith('.ios.interface') and not i.endswith('.pkg.fake')] - - + + def gen_default_suppressions(inputs, output, source_root): import collections import os supp_map = collections.defaultdict(set) - for filename in inputs: + for filename in inputs: sanitizer = os.path.basename(filename).split('.', 1)[0] with open(os.path.join(source_root, filename)) as src: for line in src: @@ -36,17 +36,17 @@ def gen_default_suppressions(inputs, output, source_root): continue supp_map[sanitizer].add(line) - with open(output, "wb") as dst: + with open(output, "wb") as dst: for supp_type, supps in supp_map.items(): dst.write('extern "C" const char *__%s_default_suppressions() {\n' % supp_type) dst.write(' return "{}";\n'.format('\\n'.join(sorted(supps)))) dst.write('}\n') -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--musl', action='store_true') +def parse_args(): + parser = optparse.OptionParser() + parser.disable_interspersed_args() + parser.add_option('--musl', action='store_true') parser.add_option('--custom-step') parser.add_option('--python') parser.add_option('--source-root') @@ -54,13 +54,13 @@ def parse_args(): parser.add_option('--linker-output') parser.add_option('--whole-archive-peers', action='append') parser.add_option('--whole-archive-libs', action='append') - return parser.parse_args() - - + return parser.parse_args() + + if __name__ == '__main__': - opts, args = parse_args() + opts, args = parse_args() - cmd = fix_cmd(opts.musl, args) + cmd = fix_cmd(opts.musl, args) cmd = ProcessWholeArchiveOption(opts.arch, opts.whole_archive_peers, opts.whole_archive_libs).construct_cmd(cmd) if opts.custom_step: diff --git a/build/scripts/run_msvc_wine.py b/build/scripts/run_msvc_wine.py index ed2660f3b8..439d1f8831 100644 --- a/build/scripts/run_msvc_wine.py +++ b/build/scripts/run_msvc_wine.py @@ -407,7 +407,7 @@ def process_free_args(args, wine, bld_root, mode): else: result.append(process_arg(arg)) return pwa.ProcessWholeArchiveOption('WINDOWS', wa_peers, wa_libs).construct_cmd(result) - + def run_main(): parser = argparse.ArgumentParser() parser.add_argument('wine', action='store') @@ -507,8 +507,8 @@ def run_main(): # non-zero return code - bad, return it immediately if rc: - print >>sys.stderr, '##win_cmd##' + ' '.join(cmd) - print >>sys.stderr, '##args##' + ' '.join(free_args) + print >>sys.stderr, '##win_cmd##' + ' '.join(cmd) + print >>sys.stderr, '##args##' + ' '.join(free_args) return rc # check for output existence(if we expect it!) and real length diff --git a/build/scripts/vcs_info.py b/build/scripts/vcs_info.py index fa7376e739..319d1b4631 100644 --- a/build/scripts/vcs_info.py +++ b/build/scripts/vcs_info.py @@ -1,16 +1,16 @@ -import base64 -import json -import os -import re -import sys -import shutil -import tempfile -import textwrap -import zipfile - - -class _Formatting(object): - @staticmethod +import base64 +import json +import os +import re +import sys +import shutil +import tempfile +import textwrap +import zipfile + + +class _Formatting(object): + @staticmethod def is_str(strval): if sys.version_info >= (3, 0, 0): return isinstance(strval, (bytes,str)) @@ -25,266 +25,266 @@ class _Formatting(object): return isinstance(strval, unicode) @staticmethod - def escape_special_symbols(strval): + def escape_special_symbols(strval): encoding_needed = _Formatting.encoding_needed(strval) c_str = strval.encode('utf-8') if encoding_needed else strval retval = b"" - for c in c_str: + for c in c_str: if sys.version_info >= (3, 0, 0): c = bytes([c]) - if c in ("\\", "\""): - retval += "\\" + c - elif ord(c) < ord(' '): + if c in ("\\", "\""): + retval += "\\" + c + elif ord(c) < ord(' '): retval += c.decode('latin-1').encode('unicode_escape') - else: - retval += c + else: + retval += c return retval.decode('utf-8') if encoding_needed else retval - - @staticmethod - def escape_line_feed(strval, indent=' '): - return strval.replace(r'\n', '\\n"\\\n' + indent + '"') - - @staticmethod + + @staticmethod + def escape_line_feed(strval, indent=' '): + return strval.replace(r'\n', '\\n"\\\n' + indent + '"') + + @staticmethod def escape_trigraphs(strval): return strval.replace(r'?', '\\?') @staticmethod - def escaped_define(strkey, val): - name = "#define " + strkey + " " + def escaped_define(strkey, val): + name = "#define " + strkey + " " if _Formatting.is_str(val): - define = "\"" + _Formatting.escape_line_feed( + define = "\"" + _Formatting.escape_line_feed( _Formatting.escape_trigraphs(_Formatting.escape_special_symbols(val))) + "\"" - else: - define = str(val) - return name + define - - @staticmethod - def escaped_go_map_key(strkey, strval): + else: + define = str(val) + return name + define + + @staticmethod + def escaped_go_map_key(strkey, strval): if _Formatting.is_str(strval): - return ' ' + '"' + strkey + '": "' + _Formatting.escape_special_symbols(strval) + '",' - else: - return ' ' + '"' + strkey + '": "' + str(strval) + '",' - - -def get_default_json(): - return json.loads('''{ + return ' ' + '"' + strkey + '": "' + _Formatting.escape_special_symbols(strval) + '",' + else: + return ' ' + '"' + strkey + '": "' + str(strval) + '",' + + +def get_default_json(): + return json.loads('''{ "ARCADIA_SOURCE_HG_HASH": "0000000000000000000000000000000000000000", "ARCADIA_SOURCE_LAST_AUTHOR": "<UNKNOWN>", - "ARCADIA_SOURCE_LAST_CHANGE": -1, - "ARCADIA_SOURCE_PATH": "/", - "ARCADIA_SOURCE_REVISION": -1, - "ARCADIA_SOURCE_URL": "", - "BRANCH": "unknown-vcs-branch", - "BUILD_DATE": "", + "ARCADIA_SOURCE_LAST_CHANGE": -1, + "ARCADIA_SOURCE_PATH": "/", + "ARCADIA_SOURCE_REVISION": -1, + "ARCADIA_SOURCE_URL": "", + "BRANCH": "unknown-vcs-branch", + "BUILD_DATE": "", "BUILD_TIMESTAMP": 0, - "BUILD_HOST": "localhost", - "BUILD_USER": "nobody", + "BUILD_HOST": "localhost", + "BUILD_USER": "nobody", "PROGRAM_VERSION": "Arc info:\\n Branch: unknown-vcs-branch\\n Commit: 0000000000000000000000000000000000000000\\n Author: <UNKNOWN>\\n Summary: No VCS\\n\\n", "SCM_DATA": "Arc info:\\n Branch: unknown-vcs-branch\\n Commit: 0000000000000000000000000000000000000000\\n Author: <UNKNOWN>\\n Summary: No VCS\\n", - "VCS": "arc", - "ARCADIA_PATCH_NUMBER": 0, - "ARCADIA_TAG": "" -}''') - - -def get_json(file_name): - try: - with open(file_name, 'r') as f: - out = json.load(f) - - # TODO: check 'tar+svn' parsing - for i in ['ARCADIA_SOURCE_REVISION', 'ARCADIA_SOURCE_LAST_CHANGE', 'SVN_REVISION']: + "VCS": "arc", + "ARCADIA_PATCH_NUMBER": 0, + "ARCADIA_TAG": "" +}''') + + +def get_json(file_name): + try: + with open(file_name, 'r') as f: + out = json.load(f) + + # TODO: check 'tar+svn' parsing + for i in ['ARCADIA_SOURCE_REVISION', 'ARCADIA_SOURCE_LAST_CHANGE', 'SVN_REVISION']: if i in out and _Formatting.is_str(out[i]): - try: - out[i] = int(out[i]) - except: - out[i] = -1 - return out - except: - return get_default_json() - - -def print_c(json_file, output_file, argv): - """ params: - json file - output file - $(SOURCE_ROOT)/build/scripts/c_templates/svn_interface.c""" - def gen_header(info): - lines = [] - for k, v in info.items(): - lines.append(_Formatting.escaped_define(k, v)) - return lines - - interface = argv[0] - - with open(interface) as c: - c_file = c.read() - with open(output_file, 'w') as f: + try: + out[i] = int(out[i]) + except: + out[i] = -1 + return out + except: + return get_default_json() + + +def print_c(json_file, output_file, argv): + """ params: + json file + output file + $(SOURCE_ROOT)/build/scripts/c_templates/svn_interface.c""" + def gen_header(info): + lines = [] + for k, v in info.items(): + lines.append(_Formatting.escaped_define(k, v)) + return lines + + interface = argv[0] + + with open(interface) as c: + c_file = c.read() + with open(output_file, 'w') as f: header = '\n'.join(gen_header(json_file)) if sys.version_info < (3, 0, 0): header = header.encode('utf-8') f.write(header + '\n' + c_file) - - -def merge_java_content(old_content, json_file): - new_content, names = print_java_mf(json_file) - - def split_to_sections(content): - sections = [] - cur_section = [] - for l in content: - if l.rstrip(): - cur_section.append(l) - else: - sections.append(cur_section) - cur_section = [] - - if cur_section: # should not be needed according to format specification - sections.append(cur_section) - - return sections - - def drop_duplicate_entries(main_section, names): - header = re.compile('^([A-Za-z0-9][A-Za-z0-9_-]*): .*$') - new_main_section = [] - for l in main_section: - match = header.match(l) - # duplicate entry - if match: - skip = match.group(1) in names - - if not skip: - new_main_section.append(l) - return new_main_section - - if old_content: - sections = split_to_sections(old_content) - sections[0] = drop_duplicate_entries(sections[0], names) - else: - sections = [['Manifest-Version: 1.0\n']] - - sections[0].extend(map(lambda x: x + '\n', new_content)) - - return ''.join(map(lambda x: ''.join(x), sections)) + '\n' - - -def merge_java_mf_jar(json_file, out_manifest, jar_file): - try: - temp_dir = tempfile.mkdtemp() - try: - with zipfile.ZipFile(jar_file, 'r') as jar: - jar.extract(os.path.join('META-INF', 'MANIFEST.MF'), path=temp_dir) - except KeyError: - pass - - merge_java_mf_dir(json_file, out_manifest, temp_dir) - finally: - shutil.rmtree(temp_dir) - - -def merge_java_mf_dir(json_file, out_manifest, input_dir): - manifest = os.path.join(input_dir, 'META-INF', 'MANIFEST.MF') - - old_lines = [] - if os.path.isfile(manifest): - with open(manifest, 'r') as f: - old_lines = f.readlines() - - with open(out_manifest, 'w') as f: - f.write(merge_java_content(old_lines, json_file)) - - -def merge_java_mf(json_file, out_manifest, input): - if zipfile.is_zipfile(input): - merge_java_mf_jar(json_file, out_manifest, input) - elif os.path.isdir(input): - merge_java_mf_dir(json_file, out_manifest, input) - - -def print_java_mf(info): - wrapper = textwrap.TextWrapper(subsequent_indent=' ', - break_long_words=True, - replace_whitespace=False, - drop_whitespace=False) - names = set() - - def wrap(key, val): - names.add(key[:-2]) - if not val: - return [] - return wrapper.wrap(key + val) - - lines = wrap('Program-Version-String: ', base64.b64encode(info['PROGRAM_VERSION'].encode('utf-8'))) - lines += wrap('SCM-String: ', base64.b64encode(info['SCM_DATA'].encode('utf-8'))) - lines += wrap('Arcadia-Source-Path: ', info['ARCADIA_SOURCE_PATH']) - lines += wrap('Arcadia-Source-URL: ', info['ARCADIA_SOURCE_URL']) - lines += wrap('Arcadia-Source-Revision: ', str(info['ARCADIA_SOURCE_REVISION'])) - lines += wrap('Arcadia-Source-Hash: ', info['ARCADIA_SOURCE_HG_HASH'].rstrip()) - lines += wrap('Arcadia-Source-Last-Change: ', str(info['ARCADIA_SOURCE_LAST_CHANGE'])) - lines += wrap('Arcadia-Source-Last-Author: ', info['ARCADIA_SOURCE_LAST_AUTHOR']) - lines += wrap('Build-User: ', info['BUILD_USER']) - lines += wrap('Build-Host: ', info['BUILD_HOST']) - lines += wrap('Version-Control-System: ', info['VCS']) - lines += wrap('Branch: ', info['BRANCH']) - lines += wrap('Arcadia-Tag: ', info.get('ARCADIA_TAG', '')) - lines += wrap('Arcadia-Patch-Number: ', str(info.get('ARCADIA_PATCH_NUMBER', 42))) - if 'SVN_REVISION' in info: - lines += wrap('SVN-Revision: ', str(info['SVN_REVISION'])) - lines += wrap('SVN-Arcroot: ', info['SVN_ARCROOT']) - lines += wrap('SVN-Time: ', info['SVN_TIME']) - lines += wrap('Build-Date: ', info['BUILD_DATE']) + + +def merge_java_content(old_content, json_file): + new_content, names = print_java_mf(json_file) + + def split_to_sections(content): + sections = [] + cur_section = [] + for l in content: + if l.rstrip(): + cur_section.append(l) + else: + sections.append(cur_section) + cur_section = [] + + if cur_section: # should not be needed according to format specification + sections.append(cur_section) + + return sections + + def drop_duplicate_entries(main_section, names): + header = re.compile('^([A-Za-z0-9][A-Za-z0-9_-]*): .*$') + new_main_section = [] + for l in main_section: + match = header.match(l) + # duplicate entry + if match: + skip = match.group(1) in names + + if not skip: + new_main_section.append(l) + return new_main_section + + if old_content: + sections = split_to_sections(old_content) + sections[0] = drop_duplicate_entries(sections[0], names) + else: + sections = [['Manifest-Version: 1.0\n']] + + sections[0].extend(map(lambda x: x + '\n', new_content)) + + return ''.join(map(lambda x: ''.join(x), sections)) + '\n' + + +def merge_java_mf_jar(json_file, out_manifest, jar_file): + try: + temp_dir = tempfile.mkdtemp() + try: + with zipfile.ZipFile(jar_file, 'r') as jar: + jar.extract(os.path.join('META-INF', 'MANIFEST.MF'), path=temp_dir) + except KeyError: + pass + + merge_java_mf_dir(json_file, out_manifest, temp_dir) + finally: + shutil.rmtree(temp_dir) + + +def merge_java_mf_dir(json_file, out_manifest, input_dir): + manifest = os.path.join(input_dir, 'META-INF', 'MANIFEST.MF') + + old_lines = [] + if os.path.isfile(manifest): + with open(manifest, 'r') as f: + old_lines = f.readlines() + + with open(out_manifest, 'w') as f: + f.write(merge_java_content(old_lines, json_file)) + + +def merge_java_mf(json_file, out_manifest, input): + if zipfile.is_zipfile(input): + merge_java_mf_jar(json_file, out_manifest, input) + elif os.path.isdir(input): + merge_java_mf_dir(json_file, out_manifest, input) + + +def print_java_mf(info): + wrapper = textwrap.TextWrapper(subsequent_indent=' ', + break_long_words=True, + replace_whitespace=False, + drop_whitespace=False) + names = set() + + def wrap(key, val): + names.add(key[:-2]) + if not val: + return [] + return wrapper.wrap(key + val) + + lines = wrap('Program-Version-String: ', base64.b64encode(info['PROGRAM_VERSION'].encode('utf-8'))) + lines += wrap('SCM-String: ', base64.b64encode(info['SCM_DATA'].encode('utf-8'))) + lines += wrap('Arcadia-Source-Path: ', info['ARCADIA_SOURCE_PATH']) + lines += wrap('Arcadia-Source-URL: ', info['ARCADIA_SOURCE_URL']) + lines += wrap('Arcadia-Source-Revision: ', str(info['ARCADIA_SOURCE_REVISION'])) + lines += wrap('Arcadia-Source-Hash: ', info['ARCADIA_SOURCE_HG_HASH'].rstrip()) + lines += wrap('Arcadia-Source-Last-Change: ', str(info['ARCADIA_SOURCE_LAST_CHANGE'])) + lines += wrap('Arcadia-Source-Last-Author: ', info['ARCADIA_SOURCE_LAST_AUTHOR']) + lines += wrap('Build-User: ', info['BUILD_USER']) + lines += wrap('Build-Host: ', info['BUILD_HOST']) + lines += wrap('Version-Control-System: ', info['VCS']) + lines += wrap('Branch: ', info['BRANCH']) + lines += wrap('Arcadia-Tag: ', info.get('ARCADIA_TAG', '')) + lines += wrap('Arcadia-Patch-Number: ', str(info.get('ARCADIA_PATCH_NUMBER', 42))) + if 'SVN_REVISION' in info: + lines += wrap('SVN-Revision: ', str(info['SVN_REVISION'])) + lines += wrap('SVN-Arcroot: ', info['SVN_ARCROOT']) + lines += wrap('SVN-Time: ', info['SVN_TIME']) + lines += wrap('Build-Date: ', info['BUILD_DATE']) if 'BUILD_TIMESTAMP' in info: lines += wrap('Build-Timestamp: ', str(info['BUILD_TIMESTAMP'])) - return lines, names - - -def print_java(json_file, output_file, argv): - """ params: - json file - output file - file""" - input = argv[0] if argv else os.curdir - merge_java_mf(json_file, output_file, input) - - -def print_go(json_file, output_file): - def gen_map(info): - lines = [] - for k, v in info.items(): - lines.append(_Formatting.escaped_go_map_key(k, v)) - return lines - - with open(output_file, 'w') as f: - f.write('\n'.join([ - 'package main', - 'import ("a.yandex-team.ru/library/go/core/buildinfo")', - 'var buildinfomap = map[string]string {'] + gen_map(json_file) + ['}'] + - ['func init() {', - ' buildinfo.InitBuildInfo(buildinfomap)', - '}'] - ).encode('utf-8') + '\n') - - -if __name__ == '__main__': - if 'output-go' in sys.argv: - lang = 'Go' - sys.argv.remove('output-go') - elif 'output-java' in sys.argv: - lang = 'Java' - sys.argv.remove('output-java') - else: - lang = 'C' - - if 'no-vcs' in sys.argv: - sys.argv.remove('no-vcs') - json_file = get_default_json() - else: - json_name = sys.argv[1] - json_file = get_json(json_name) - - if lang == 'Go': - print_go(json_file, sys.argv[2]) - elif lang == 'Java': - print_java(json_file, sys.argv[2], sys.argv[3:]) - else: - print_c(json_file, sys.argv[2], sys.argv[3:]) + return lines, names + + +def print_java(json_file, output_file, argv): + """ params: + json file + output file + file""" + input = argv[0] if argv else os.curdir + merge_java_mf(json_file, output_file, input) + + +def print_go(json_file, output_file): + def gen_map(info): + lines = [] + for k, v in info.items(): + lines.append(_Formatting.escaped_go_map_key(k, v)) + return lines + + with open(output_file, 'w') as f: + f.write('\n'.join([ + 'package main', + 'import ("a.yandex-team.ru/library/go/core/buildinfo")', + 'var buildinfomap = map[string]string {'] + gen_map(json_file) + ['}'] + + ['func init() {', + ' buildinfo.InitBuildInfo(buildinfomap)', + '}'] + ).encode('utf-8') + '\n') + + +if __name__ == '__main__': + if 'output-go' in sys.argv: + lang = 'Go' + sys.argv.remove('output-go') + elif 'output-java' in sys.argv: + lang = 'Java' + sys.argv.remove('output-java') + else: + lang = 'C' + + if 'no-vcs' in sys.argv: + sys.argv.remove('no-vcs') + json_file = get_default_json() + else: + json_name = sys.argv[1] + json_file = get_json(json_name) + + if lang == 'Go': + print_go(json_file, sys.argv[2]) + elif lang == 'Java': + print_java(json_file, sys.argv[2], sys.argv[3:]) + else: + print_c(json_file, sys.argv[2], sys.argv[3:]) diff --git a/build/scripts/ya.make b/build/scripts/ya.make index a4830bec08..710165e40d 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -54,7 +54,7 @@ TEST_SRCS( java_pack_to_file.py link_asrc.py link_dyn_lib.py - link_exe.py + link_exe.py link_fat_obj.py link_lib.py llvm_opt_wrapper.py @@ -83,7 +83,7 @@ TEST_SRCS( tared_protoc.py touch.py unpacking_jtest_runner.py - vcs_info.py + vcs_info.py with_coverage.py with_crash_on_timeout.py with_pathsep_resolve.py diff --git a/build/ya.conf.json b/build/ya.conf.json index 69c2d86ff0..5f7cc875d6 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -4646,14 +4646,14 @@ } ] }, - "ya-tc": { - "tools": { + "ya-tc": { + "tools": { "ya-tc": { "bottle": "ya-tc", "executable": "ya-tc" } - }, - "platforms": [ + }, + "platforms": [ { "host": { "os": "LINUX" @@ -4666,8 +4666,8 @@ }, "default": true } - ] - }, + ] + }, "crypta": { "tools": { "crypta": { @@ -7304,17 +7304,17 @@ ] } }, - "ya-tc": { - "formula": { + "ya-tc": { + "formula": { "sandbox_id": 981372621, - "match": "ya-tc" - }, - "executable": { + "match": "ya-tc" + }, + "executable": { "ya-tc": [ "ya-tc" ] - } - }, + } + }, "crypta": { "formula": { "sandbox_id": 550700694, diff --git a/build/ya.make b/build/ya.make index e9a5f2bb4d..407b8f13e0 100644 --- a/build/ya.make +++ b/build/ya.make @@ -22,7 +22,7 @@ END() RECURSE( conf_fatal_error config - docs/empty + docs/empty external_resources platform/java platform/local_so diff --git a/build/ymake.core.conf b/build/ymake.core.conf index c0a48d946b..081833998b 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -437,7 +437,7 @@ when ($SANITIZER_TYPE) { # tag:internal ### @usage: CHECK_CONFIG_H(<conf_header>) # internal ### -### This internal macro adds checking code for configuration header in external (contrib) library. +### This internal macro adds checking code for configuration header in external (contrib) library. ### The check is needed to avoid conflicts on certain types and functions available in arcadia. ### ### @see https://a.yandex-team.ru/arc/trunk/arcadia/build/scripts/check_config_h.py for exact details @@ -928,8 +928,8 @@ when ($JAVA_PROTO_RUNTIME == "javalite") { otherwise { _JAVA_PROTO_OUT_ARG=--java_out= JAVA_PROTOBUF_PEERS=contrib/java/com/google/protobuf/protobuf-java/${JAVA_PROTO_RUNTIME_VERSION} - JAVA_GRPC_STUB=contrib/java/io/grpc/grpc-stub/1.26.0 - JAVA_GRPC_PROTOBUF=contrib/java/io/grpc/grpc-protobuf/1.26.0 + JAVA_GRPC_STUB=contrib/java/io/grpc/grpc-stub/1.26.0 + JAVA_GRPC_PROTOBUF=contrib/java/io/grpc/grpc-protobuf/1.26.0 } # tag:proto tag:java-specific @@ -952,7 +952,7 @@ macro XS_PROTO(File, Dir, Outputs...) { # tag:proto tag:fbs ### @usage: PROTO2FBS(InputProto) ### -### Produce flatbuf schema out of protobuf description. +### Produce flatbuf schema out of protobuf description. macro PROTO2FBS(File) { .CMD=${cwd:BINDIR} ${tool:"contrib/tools/flatc"} -I . -I ${ARCADIA_ROOT} --proto ${input:File} ${output;hide;nopath;noext:File.fbs} ${kv;hide:"p FBS"} ${kv;hide:"pc yellow"} } @@ -1290,8 +1290,8 @@ module _BASE_UNIT: _BARE_UNIT { when ($MUSL == "yes") { CFLAGS += -D_musl_ - LINK_DYN_LIB_FLAGS += --musl - LINK_SCRIPT_EXE_FLAGS += --musl + LINK_DYN_LIB_FLAGS += --musl + LINK_SCRIPT_EXE_FLAGS += --musl PEERDIR+=contrib/libs/musl/include } @@ -1656,10 +1656,10 @@ module _BASE_PROGRAM: _LINK_UNIT { when ($CLANG_COVERAGE && $CLANG_COVERAGE != "no") { PEERDIR+=library/cpp/testing/dump_clang_coverage } - - when ($IDE_MSVS == "yes") { - PEERDIR+=build/scripts/c_templates - } + + when ($IDE_MSVS == "yes") { + PEERDIR+=build/scripts/c_templates + } when ($_CUSTOM_LINK_STEP_SCRIPT) { LINK_SCRIPT_EXE_FLAGS+=--python=$YMAKE_PYTHON --custom-step=${input:_CUSTOM_LINK_STEP_SCRIPT} @@ -2037,7 +2037,7 @@ TEST_DATA_VALUE= ### Specifies the path to the data necessary test. ### Valid values are: arcadia/<path> , arcadia_tests_data/<path> and sbr://<resource_id>. ### In the latter case resource will be brought to the working directory of the test before it is started -### +### ### Used only inside TEST modules. ### ### Documentation: https://wiki.yandex-team.ru/yatool/test/#dannyeizrepozitorija @@ -2268,7 +2268,7 @@ module GTEST_UGLY: _BASE_PROGRAM { ### ### @example: ### -### EXECTEST() +### EXECTEST() ### OWNER(g:yatool) ### ### RUN( @@ -2940,9 +2940,9 @@ multimodule DYNAMIC_LIBRARY { ### @example: ### Consider the file to ya.make: ### -### LIBRARY() +### LIBRARY() ### GLOBAL_SRCS(foo.cpp bar.cpp) -### END() +### END() ### ### @see: [SRCS()](#macro_SRCS) macro GLOBAL_SRCS(Files...) { @@ -2957,10 +2957,10 @@ macro GLOBAL_SRCS(Files...) { ### ### @example: ### -### LIBRARY() +### LIBRARY() ### SET(MAKE_IT_GLOBAL GLOBAL) ### ALL_SRCS(${MAKE_IT_GLOBAL} foo.cpp bar.cpp) -### END() +### END() ### ### @see: [GLOBAL_SRCS()](#macro_GLOBAL_SRCS), [SRCS()](#macro_SRCS) macro ALL_SRCS(GLOBAL?"GLOBAL":"", Files...) { @@ -3405,7 +3405,7 @@ macro EMBED_JAVA_VCS_INFO() { ### @usage: EXTERNAL_JAVA_LIBRARY() #internal ### ### EXTERNAL_JAVA_LIBRARY() is a module for creating a .jar file using non-Java code (generators etc.) -### Unlike regular JAVA_LIBRARY this module doesn't produce .pom file, so it cannot be exported to Maven itself. +### Unlike regular JAVA_LIBRARY this module doesn't produce .pom file, so it cannot be exported to Maven itself. ### PEERDIR it from JAVA_LIBRARY or JAVA_PROGRAM for export to Maven. module EXTERNAL_JAVA_LIBRARY: _BASE_UNIT { .EXTS=.jsrc .java .jar .mf @@ -4491,13 +4491,13 @@ macro TEST_CWD(Arg) { MAKE_UBERJAR_VALUE= ### @usage: UBERJAR() ### -### UBERJAR is a single all-in-one jar-archive that includes all its Java dependencies (reachable PEERDIR). +### UBERJAR is a single all-in-one jar-archive that includes all its Java dependencies (reachable PEERDIR). ### It also supports shading classes inside the archive by moving them to a different package (similar to the maven-shade-plugin). ### Use UBERJAR inside JAVA_PROGRAM module. ### ### You can use the following macros to configure the archive: ### 1. UBERJAR_HIDING_PREFIX prefix for classes to shade (classes remain in their packages by default) -### 2. UBERJAR_HIDE_EXCLUDE_PATTERN exclude classes matching this patterns from shading (if enabled). +### 2. UBERJAR_HIDE_EXCLUDE_PATTERN exclude classes matching this patterns from shading (if enabled). ### 3. UBERJAR_PATH_EXCLUDE_PREFIX the prefix for classes that should not get into the jar archive (all classes are placed into the archive by default) ### 4. UBERJAR_MANIFEST_TRANSFORMER_MAIN add ManifestResourceTransformer class to uberjar processing and specify main-class ### 5. UBERJAR_MANIFEST_TRANSFORMER_ATTRIBUTE add ManifestResourceTransformer class to uberjar processing and specify some attribute @@ -4708,15 +4708,15 @@ JAVA_SRCS_VALUE= ### Keywords: ### 1. X SRCDIR - specify the directory x is performed relatively to search the source code for these patterns. If there is no SRCDIR, the source will be searched relative to the module directory. ### 2. PACKAGE_PREFIX x - use if source paths relative to the SRCDIR does not coincide with the full class names. For example, if all sources of module are in the same package, you can create a directory package/name , and just put the source code in the SRCDIR and specify PACKAGE_PREFIX package.name. -### +### ### @example: -### - example/ya.make +### - example/ya.make ### ### JAVA_PROGRAM() ### JAVA_SRCS(SRCDIR src/main/java **/*) ### END() ### -### - example/src/main/java/ru/yandex/example/HelloWorld.java +### - example/src/main/java/ru/yandex/example/HelloWorld.java ### ### package ru.yandex.example; ### public class HelloWorld { @@ -4753,15 +4753,15 @@ TEST_CLASSPATH_MANAGED= # tag:java-specific DEPENDENCY_MANAGEMENT_VALUE= -### @usage: DEPENDENCY_MANAGEMENT(path/to/lib1 path/to/lib2 ...) -### +### @usage: DEPENDENCY_MANAGEMENT(path/to/lib1 path/to/lib2 ...) +### ### Lock version of the library from the contrib/java at some point, so that all unversioned PEERDIRs to this library refer to the specified version. ### ### For example, if the moduse has PEERDIR (contrib/java/junit/junit), and ### 1. specifies DEPENDENCY_MANAGEMENT(contrib/java/junit/junit/4.12), ### the PEERDIR is automatically replaced by contrib/java/junit/junit/4.12; ### 2. doesn't specify DEPENDENCY_MANAGEMENT, PEERDIR automatically replaced -### with the default from contrib/java/junit/junit/ya.make. +### with the default from contrib/java/junit/junit/ya.make. ### These defaults are always there and are supported by maven-import, which puts ### there the maximum version available in contrib/java. ### @@ -4988,7 +4988,7 @@ macro JAR_EXCLUDE(Filters...) { ### The final artifact is provided when SANDBOX_TASK is referred to by DEPENDS and BUNDLE macros. ### As PEERDIR target, it works like regular PY2_LIBRARY with predefined dependencies on Sandbox SDK to allow code reuse among SANDBOX_TASKs. ### -### Currently Sandbox supports only Python 2.x, so both variants will be compatible only with Python 2.x modules +### Currently Sandbox supports only Python 2.x, so both variants will be compatible only with Python 2.x modules ### and will select multimodule variants accordingly. ### ### Documentation: https://wiki.yandex-team.ru/sandbox/tasks/binary @@ -5263,19 +5263,19 @@ macro ADD_PERL_MODULE(Dir, Module) { ### - LF_YT - Allocator selection for YT (https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/lfalloc/yt/ya.make) ### - LF_DBG - Debug allocator selection (https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/lfalloc/dbg/ya.make) ### - YT - The YTAlloc allocator (https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/ytalloc/impl/ya.make) -### - J - The JEMalloc allocator (https://a.yandex-team.ru/arc/trunk/arcadia/library/malloc/jemalloc) -### - B - The balloc allocator named Pyotr Popov and Anton Samokhvalov -### - Discussion: https://ironpeter.at.yandex-team.ru/replies.xml?item_no=126 +### - J - The JEMalloc allocator (https://a.yandex-team.ru/arc/trunk/arcadia/library/malloc/jemalloc) +### - B - The balloc allocator named Pyotr Popov and Anton Samokhvalov +### - Discussion: https://ironpeter.at.yandex-team.ru/replies.xml?item_no=126 ### - Code: https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/balloc -### - BM - The balloc for market (agri@ commits from july 2018 till November 2018 saved) -### - C - Like B, but can be disabled for each thread to LF or SYSTEM one (B can be disabled only to SYSTEM) +### - BM - The balloc for market (agri@ commits from july 2018 till November 2018 saved) +### - C - Like B, but can be disabled for each thread to LF or SYSTEM one (B can be disabled only to SYSTEM) ### - MIM - Microsoft's mimalloc (actual version) (https://a.yandex-team.ru/arc/trunk/arcadia/library/malloc/mimalloc) ### - TCMALLOC - Google TCMalloc (actual version) (https://a.yandex-team.ru/arc/trunk/arcadia/library/malloc/tcmalloc) -### - GOOGLE - Google TCMalloc (https://a.yandex-team.ru/arc/trunk/arcadia/library/malloc/galloc) -### - LOCKLESS - Allocator based upon lockless queues (https://a.yandex-team.ru/arc/trunk/arcadia/library/malloc/lockless) -### - SYSTEM - Use target system allocator -### - FAKE - Don't link with any allocator -### +### - GOOGLE - Google TCMalloc (https://a.yandex-team.ru/arc/trunk/arcadia/library/malloc/galloc) +### - LOCKLESS - Allocator based upon lockless queues (https://a.yandex-team.ru/arc/trunk/arcadia/library/malloc/lockless) +### - SYSTEM - Use target system allocator +### - FAKE - Don't link with any allocator +### ### More about allocators in Arcadia: https://wiki.yandex-team.ru/arcadia/allocators/ macro ALLOCATOR(Alloc) { SET(ALLOCATOR $Alloc) @@ -5584,7 +5584,7 @@ macro JOIN_SRCS(Out, Src...) { ### ### Join set of sources into single file named Out and send it for further processing as if it were listed as SRCS(GLOBAL Out). ### This macro doesn't place all file into Out, it emits #include<Src>... Use the for C++ source files only. -### You should specify file name with the extension as Out. Further processing will be done according to this extension. +### You should specify file name with the extension as Out. Further processing will be done according to this extension. macro JOIN_SRCS_GLOBAL(Out, Src...) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/gen_join_srcs.py"} ${output;noauto:Out} --ya-start-command-file ${input;rootrel:Src} --ya-end-command-file ${output_include;hide:Src} ${kv;hide:"p JS"} ${kv;hide:"pc magenta"} SRCS(GLOBAL $Out) @@ -5594,7 +5594,7 @@ macro JOIN_SRCS_GLOBAL(Out, Src...) { ### ### Join set of sources into single file named Out and send it for further processing as if it were listed as SRCS(GLOBAL Out). ### This macro places all files into single file, so will work with any sources. -### You should specify file name with the extension as Out. Further processing will be done according to this extension. +### You should specify file name with the extension as Out. Further processing will be done according to this extension. macro FLAT_JOIN_SRCS_GLOBAL(Out, Src...) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/fs_tools.py"} cat ${output;noauto:Out} --ya-start-command-file ${input:Src} --ya-end-command-file ${output_include;hide:Src} ${kv;hide:"p JS"} ${kv;hide:"pc magenta"} SRCS(GLOBAL $Out) @@ -5812,7 +5812,7 @@ macro _SRC("asp", SRC, SRCFLAGS...) { # tag:src-processing macro _SRC("rl6", SRC, SRCFLAGS...) { - .CMD=$RUN_NO_SANITIZE ${tool:"contrib/tools/ragel6"} $RAGEL6_FLAGS ${SRCFLAGS} -L -I${ARCADIA_ROOT} -o ${output;nopath;noext;defext=.rl6.cpp:SRC} ${input:SRC} ${kv;hide:"p R6"} ${kv;hide:"pc yellow"} + .CMD=$RUN_NO_SANITIZE ${tool:"contrib/tools/ragel6"} $RAGEL6_FLAGS ${SRCFLAGS} -L -I${ARCADIA_ROOT} -o ${output;nopath;noext;defext=.rl6.cpp:SRC} ${input:SRC} ${kv;hide:"p R6"} ${kv;hide:"pc yellow"} .SEM=target_ragel_lexers PRIVATE ${input:SRC} ${output;hide;suf=.o:SRC} $RAGEL6_FLAGS ${SRCFLAGS} && set_global_flags RAGEL_FLAGS -L -I $S/ && conan_require ragel/6.10 && conan_import '"bin, ragel* -> ./bin"' && modules_required yandex_common.cmake } @@ -6153,9 +6153,9 @@ macro SRC(FILE, FLAGS...) { ### ### @example: ### -### LIBRARY(test_global) +### LIBRARY(test_global) ### SRCS(GLOBAL foo.cpp) -### END() +### END() ### ### This will produce foo.o and feed it to any PROGRAM/DLL module transitively depending on test_global library. The library itself will be empty and won't produce .a file. macro SRCS(FILES...) { @@ -6446,20 +6446,20 @@ VCS_C_OBJ_RR=$MODDIR/__vcs_version__.c$OBJECT_SUF VCS_C=$BINDIR/__vcs_version__.c VCS_GO=$BINDIR/__vcs_version__.go VCS_JAVA=$BINDIR/__vcs_version__.mf -VCS_INFO_DISABLE_CACHE__NO_UID__= -YASM_DEBUG_INFO_DISABLE_CACHE__NO_UID__= -CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__= -CL_MACRO_INFO_DISABLE_CACHE__NO_UID__= +VCS_INFO_DISABLE_CACHE__NO_UID__= +YASM_DEBUG_INFO_DISABLE_CACHE__NO_UID__= +CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__= +CL_MACRO_INFO_DISABLE_CACHE__NO_UID__= YASM_DEBUG_INFO= CL_DEBUG_INFO= CL_MACRO_INFO= -when ($FORCE_VCS_INFO_UPDATE == "yes") { - VCS_INFO_DISABLE_CACHE__NO_UID__=${hide;kv:"disable_cache"} -} -### Works for pyton too. -GENERATE_VCS_C_INFO_NODEP=$YMAKE_PYTHON ${input:"build/scripts/vcs_info.py"} $(VCS)/vcs.json $VCS_C ${input:"build/scripts/c_templates/svn_interface.c"} $VCS_INFO_DISABLE_CACHE__NO_UID__ && $_SRC_c_nodeps($VCS_C, $VCS_C_OBJ, $(SOURCE_ROOT)) -GENERATE_VCS_GO_INFO_NODEP=$YMAKE_PYTHON ${input:"build/scripts/vcs_info.py"} output-go $(VCS)/vcs.json $VCS_GO $VCS_INFO_DISABLE_CACHE__NO_UID__ -GENERATE_VCS_JAVA_INFO_NODEP=$YMAKE_PYTHON ${input:"build/scripts/vcs_info.py"} output-java $(VCS)/vcs.json $VCS_JAVA $VCS_INFO_DISABLE_CACHE__NO_UID__ +when ($FORCE_VCS_INFO_UPDATE == "yes") { + VCS_INFO_DISABLE_CACHE__NO_UID__=${hide;kv:"disable_cache"} +} +### Works for pyton too. +GENERATE_VCS_C_INFO_NODEP=$YMAKE_PYTHON ${input:"build/scripts/vcs_info.py"} $(VCS)/vcs.json $VCS_C ${input:"build/scripts/c_templates/svn_interface.c"} $VCS_INFO_DISABLE_CACHE__NO_UID__ && $_SRC_c_nodeps($VCS_C, $VCS_C_OBJ, $(SOURCE_ROOT)) +GENERATE_VCS_GO_INFO_NODEP=$YMAKE_PYTHON ${input:"build/scripts/vcs_info.py"} output-go $(VCS)/vcs.json $VCS_GO $VCS_INFO_DISABLE_CACHE__NO_UID__ +GENERATE_VCS_JAVA_INFO_NODEP=$YMAKE_PYTHON ${input:"build/scripts/vcs_info.py"} output-java $(VCS)/vcs.json $VCS_JAVA $VCS_INFO_DISABLE_CACHE__NO_UID__ macro UPDATE_VCS_JAVA_INFO_NODEP(Jar) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/vcs_info.py"} output-java $(VCS)/vcs.json $VCS_JAVA $Jar $VCS_INFO_DISABLE_CACHE__NO_UID__ } @@ -6494,7 +6494,7 @@ macro CFG_VARS() { ### @usage: CONFIGURE_FILE(from to) ### ### Copy file with the replacement of configuration variables in form of @ANY_CONF_VAR@ with their values. -### The values are collected during configure stage, while replacement itself happens during build stage. +### The values are collected during configure stage, while replacement itself happens during build stage. ### Used implicitly for .in-files processing. macro CONFIGURE_FILE(Src, Dst) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/configure_file.py"} ${input:Src} ${output;addincl:Dst} $CFG_VARS ${kv;hide:"p CF"} ${kv;hide:"pc yellow"} @@ -6610,7 +6610,7 @@ macro NO_OPTIMIZE() { } ### @usage: NO_COMPILER_WARNINGS() -### Disable all compiler warnings in the module. +### Disable all compiler warnings in the module. ### Priorities: NO_COMPILER_WARNINGS > NO_WERROR > WERROR_MODE > WERROR. macro NO_COMPILER_WARNINGS() { ENABLE(NO_COMPILER_WARNINGS) @@ -6638,15 +6638,15 @@ macro NO_WSHADOW() { } # tag:internal -### @usage: NO_PLATFORM_RESOURCES() # internal -### Exclude dependency on platform resources libraries. +### @usage: NO_PLATFORM_RESOURCES() # internal +### Exclude dependency on platform resources libraries. ### Most probably you'll never need this. If you think you need, please contact devtools@ for assistance. macro NO_PLATFORM_RESOURCES() { ENABLE(NOPLATFORM_RESOURCES) } # tag:internal tag:codenav -### @usage: NO_CODENAVIGATION() # internal +### @usage: NO_CODENAVIGATION() # internal ### Disable codenaviagtion for a module. Needed to avoid PEERDIR loops in codenavigation support. ### Most probably you'll never need this. If you think you need, please contact devtools@ for assistance. macro NO_CODENAVIGATION() { @@ -6714,7 +6714,7 @@ macro USE_UTIL() { # tag:deprecated ### @usage: NO_JOIN_SRC() # deprecated, does-nothing -### This macro currently does nothing. This is default behavior which cannot be overridden at module level. +### This macro currently does nothing. This is default behavior which cannot be overridden at module level. macro NO_JOIN_SRC() { ENABLE(UNUSED_MACRO) } @@ -6945,18 +6945,18 @@ SCHEEME2_STRUCT_INFO_FLAGS=-f "const static ui32 RecordSig" -u "RecordSig" --gcc ### 2. Use an environment variable - DATAWORK_SCHEEME_EXPORT_FLAGS that allows to specify flags to tools/structparser ### ### @example: -### -### SET(DATAWORK_SCHEEME_EXPORT_FLAGS --final_only -m "::") -### +### +### SET(DATAWORK_SCHEEME_EXPORT_FLAGS --final_only -m "::") +### ### all options are passed to structparser (in this example --final_only - do not export heirs with public base that contains the required field,,- m "::" only from the root namespace) ### sets in extra option ### ### @example: -### +### ### SET(EXTRACT_STRUCT_INFO_FLAGS -f \"const static ui32 RecordSig\" ### -u \"RecordSig\" -n${scheeme_name}SchemeInfo ----gcc44_no_typename no_complex_overloaded_func_export -### ${DATAWORK_SCHEEME_EXPORT_FLAGS}) -### +### ${DATAWORK_SCHEEME_EXPORT_FLAGS}) +### ### for compatibility with C++ compiler and the external environment. ### See tools/structparser for more details. macro GEN_SCHEEME2(ScheemeName, FromFile) { @@ -6988,7 +6988,7 @@ macro SYMLINK(From, To) { ### ### For absolute paths use ${ARCADIA_ROOT} and ${ARCADIA_BUILD_ROOT}, or ### ${CURDIR} and ${BINDIR} which are expanded where the outputs are used. -### Note that Tool is always built for the host platform, so be careful to provide that tool can be built for all Arcadia major host platforms (Linux, MacOS and Windows). +### Note that Tool is always built for the host platform, so be careful to provide that tool can be built for all Arcadia major host platforms (Linux, MacOS and Windows). macro RUN_PROGRAM(Tool, IN{input}[], OUT{output}[], OUT_NOAUTO{output}[], TOOL{tool}[], OUTPUT_INCLUDES[], IN_DEPS[], STDOUT="", STDOUT_NOAUTO="", CWD="", ENV[], REQUIREMENTS[], Args...) { .CMD=${cwd:CWD} ${env:ENV} ${tool:Tool} $Args ${input;hide:IN} ${input;hide:IN_DEPS} ${output_include;hide:OUTPUT_INCLUDES} ${tool;hide:TOOL} ${output;hide:OUT} ${output;noauto;hide:OUT_NOAUTO} ${output;stdout:STDOUT} ${output;stdout;noauto:STDOUT_NOAUTO} ${requirements;hide:REQUIREMENTS} ${requirements;hide:"network:restricted"} ${kv;hide:"p PR"} ${kv;hide:"pc yellow"} ${kv;hide:"show_out"} } @@ -7050,7 +7050,7 @@ macro _RUN_JAVA(IN{input}[], OUT{output}[], OUT_NOAUTO{output}[], OUTPUT_INCLUDE ### @usage: FROM_SANDBOX([FILE] resource_id [AUTOUPDATED script] [RENAME <resource files>] OUT_[NOAUTO] <output files> [EXECUTABLE] [OUTPUT_INCLUDES <include files>]) ### ### Download the resource from the Sandbox, unpack (if not explicitly specified word FILE) and add OUT files to the build. EXECUTABLE makes them executable. -### You may specify extra dependencies that output files bring using OUTPUT_INCLUDES. The change of these may e.g. lead to recompilation of .cpp files extracted from resource. +### You may specify extra dependencies that output files bring using OUTPUT_INCLUDES. The change of these may e.g. lead to recompilation of .cpp files extracted from resource. ### If there is no default processing for OUT files or you need process them specially use OUT_NOAUTO instead of OUT. ### ### It is disallowed to specify directory as OUT/OUT_NOAUTO since all outputs of commands shall be known to build system. @@ -7063,7 +7063,7 @@ macro _RUN_JAVA(IN{input}[], OUT{output}[], OUT_NOAUTO{output}[], OUTPUT_INCLUDE ### RENAME RESOURCE allows to rename the resource without specifying its file name. ### ### If AUTOUPDATED is specified than macro will be regularly updated according to autoupdate script. The dedicated Sandbox task scans the arcadia and -### changes resource_ids in such macros if newer resource of specified type is available. Note that the task seeks AUTOUPDATED in specific position, +### changes resource_ids in such macros if newer resource of specified type is available. Note that the task seeks AUTOUPDATED in specific position, ### so you shall place it immediately after resource_id. macro FROM_SANDBOX(Id, OUT{output}[], OUT_NOAUTO{output}[], OUTPUT_INCLUDES[], FILE?"--copy-to-dir":"--untar-to", AUTOUPDATED="", PREFIX=".", RENAME[], EXECUTABLE?"--executable":"", SBR="sbr:", REQUIREMENTS[]) { .CMD=${hide:SANDBOX_FAKEID} ${cwd:BINDIR} ${resource;pre=$SBR:Id} $YMAKE_PYTHON ${input:"build/scripts/fetch_from_sandbox.py"} --resource-file $(RESOURCE_ROOT)/sbr/$Id/resource --resource-id $Id $FILE $PREFIX ${pre=--rename :RENAME} $EXECUTABLE -- $OUT $OUT_NOAUTO ${input;hide:"build/scripts/fetch_from.py"} ${output_include;hide:OUTPUT_INCLUDES} ${output;hide:OUT} ${output;noauto;hide:OUT_NOAUTO} ${requirements;hide:REQUIREMENTS} ${requirements;hide:"network:full"} ${kv;hide:"p SB"} ${kv;hide:"pc yellow"} ${kv;hide:"show_out"} @@ -7983,7 +7983,7 @@ CGO2_LDFLAGS_VALUE= # tag:go-specific GO_ASM_FLAGS_VALUE= ### @usage: GO_ASM_FLAGS(flags) -### Add the specified flags to the go asm compile command line. +### Add the specified flags to the go asm compile command line. macro GO_ASM_FLAGS(Flags...) { SET_APPEND(GO_ASM_FLAGS_VALUE $Flags) } @@ -8364,7 +8364,7 @@ otherwise { # tag:go-specific tag:internal ### @usage _GO_BASE_UNIT # internal ### -### The base module of all golang modules. Defines common properties, dependencies and rules for go build. +### The base module of all golang modules. Defines common properties, dependencies and rules for go build. module _GO_BASE_UNIT: _BASE_UNIT { .EXTS=.go .gosrc .o .obj .a .symabis .mf .CMD=TOUCH_UNIT @@ -8713,7 +8713,7 @@ macro JAVA_IGNORE_CLASSPATH_CLASH_FOR(Args...) { # tag:python-specific ### @usage: PY_SRCS({| CYTHON_C} { | TOP_LEVEL | NAMESPACE ns} Files...) ### -### Build specified Python sources according to Arcadia binary Python build. Basically creates precompiled and source resources keyed with module paths. +### Build specified Python sources according to Arcadia binary Python build. Basically creates precompiled and source resources keyed with module paths. ### The resources eventually are linked into final program and can be accessed as regular Python modules. ### This custom loader linked into the program will add them to sys.meta_path. ### @@ -8806,7 +8806,7 @@ multimodule PY23_LIBRARY { # tag:python-specific ### @usage: PY23_NATIVE_LIBRARY([name]) ### -### Build LIBRARY compatible with either Python 2.x or Python 3.x depending on incoming PEERDIR. +### Build LIBRARY compatible with either Python 2.x or Python 3.x depending on incoming PEERDIR. ### ### This multimodule doesn't depend on Arcadia Python binary build. It is intended only for C++ code and cannot contain PY_SRCS and USE_PYTHON2 macros. ### Use these multimodule instead of PY23_LIBRARY if the C++ extension defined in it will be used in PY2MODULE. diff --git a/build/ymake_conf.py b/build/ymake_conf.py index 6b3c2df8f2..30219eb85e 100755 --- a/build/ymake_conf.py +++ b/build/ymake_conf.py @@ -556,7 +556,7 @@ class Build(object): @property def with_ndebug(self): - return self.build_type in ('release', 'minsizerel', 'valgrind-release', 'profile', 'gprof', 'debugnoasserts') + return self.build_type in ('release', 'minsizerel', 'valgrind-release', 'profile', 'gprof', 'debugnoasserts') @property def is_valgrind(self): @@ -1547,11 +1547,11 @@ class GnuCompiler(Compiler): FSTACK += -fstack-check }''') - c_builtins = [ + c_builtins = [ "-Wno-builtin-macro-redefined", '-D__DATE__=\\""Sep 31 2019\\""', '-D__TIME__=\\"00:00:00\\"', - ] + ] compiler_supports_macro_prefix_map = ( self.tc.is_clang and self.tc.version_at_least(10) or self.tc.is_gcc and self.tc.version_at_least(8) @@ -1568,31 +1568,31 @@ class GnuCompiler(Compiler): # XXX this macro substitution breaks __FILE__ in included sources '-D__FILE__=\\""${input;qe;rootrel:SRC}\\""', ] - c_debug_map = [ - # XXX does not support non-normalized paths - "-fdebug-prefix-map=${ARCADIA_BUILD_ROOT}=/-B", - "-fdebug-prefix-map=${ARCADIA_ROOT}=/-S", - "-fdebug-prefix-map=$(TOOL_ROOT)=/-T", - ] - c_debug_map_cl = c_debug_map + [ - "-Xclang", "-fdebug-compilation-dir", "-Xclang", "/tmp", - ] - c_debug_map_light = [ - # XXX does not support non-normalized paths - "-fdebug-prefix-map=${ARCADIA_BUILD_ROOT}=/-B", - ] - c_debug_map_light_cl = c_debug_map_light + [ - "-Xclang", "-fdebug-compilation-dir", "-Xclang", "/tmp", - ] - yasm_debug_map = [ - # XXX does not support non-normalized paths - "--replace=${ARCADIA_BUILD_ROOT}=/-B", - "--replace=${ARCADIA_ROOT}=/-S", - "--replace=$(TOOL_ROOT)=/-T" - ] - emit_big(''' + c_debug_map = [ + # XXX does not support non-normalized paths + "-fdebug-prefix-map=${ARCADIA_BUILD_ROOT}=/-B", + "-fdebug-prefix-map=${ARCADIA_ROOT}=/-S", + "-fdebug-prefix-map=$(TOOL_ROOT)=/-T", + ] + c_debug_map_cl = c_debug_map + [ + "-Xclang", "-fdebug-compilation-dir", "-Xclang", "/tmp", + ] + c_debug_map_light = [ + # XXX does not support non-normalized paths + "-fdebug-prefix-map=${ARCADIA_BUILD_ROOT}=/-B", + ] + c_debug_map_light_cl = c_debug_map_light + [ + "-Xclang", "-fdebug-compilation-dir", "-Xclang", "/tmp", + ] + yasm_debug_map = [ + # XXX does not support non-normalized paths + "--replace=${ARCADIA_BUILD_ROOT}=/-B", + "--replace=${ARCADIA_ROOT}=/-S", + "--replace=$(TOOL_ROOT)=/-T" + ] + emit_big(''' when ($FORCE_CONSISTENT_DEBUG == "yes") {{ - when ($CLANG == "yes") {{ + when ($CLANG == "yes") {{ CL_DEBUG_INFO={c_debug_cl} }} otherwise {{ @@ -1602,37 +1602,37 @@ class GnuCompiler(Compiler): }} elsewhen ($CONSISTENT_DEBUG == "yes") {{ when ($CLANG == "yes") {{ - CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__={c_debug_cl} - }} - otherwise {{ - CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__={c_debug} - }} - YASM_DEBUG_INFO_DISABLE_CACHE__NO_UID__={yasm_debug} - }} - elsewhen ($CONSISTENT_DEBUG_LIGHT == "yes") {{ - when ($CLANG == "yes") {{ - CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__={c_debug_light_cl} - }} - otherwise {{ - CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__={c_debug_light} - }} - YASM_DEBUG_INFO_DISABLE_CACHE__NO_UID__={yasm_debug_light} - }} - + CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__={c_debug_cl} + }} + otherwise {{ + CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__={c_debug} + }} + YASM_DEBUG_INFO_DISABLE_CACHE__NO_UID__={yasm_debug} + }} + elsewhen ($CONSISTENT_DEBUG_LIGHT == "yes") {{ + when ($CLANG == "yes") {{ + CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__={c_debug_light_cl} + }} + otherwise {{ + CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__={c_debug_light} + }} + YASM_DEBUG_INFO_DISABLE_CACHE__NO_UID__={yasm_debug_light} + }} + when ($FORCE_CONSISTENT_BUILD == "yes") {{ CL_MACRO_INFO={macro} }} elsewhen ($CONSISTENT_BUILD == "yes") {{ - CL_MACRO_INFO_DISABLE_CACHE__NO_UID__={macro} - }} - '''.format(c_debug=' '.join(c_debug_map), - c_debug_cl=' '.join(c_debug_map_cl), - yasm_debug=' '.join(yasm_debug_map), - c_debug_light=' '.join(c_debug_map_light), # build_root substitution only - c_debug_light_cl=' '.join(c_debug_map_light_cl), # build_root substitution only - yasm_debug_light=yasm_debug_map[0], # build_root substitution only - macro=' '.join(c_builtins))) - + CL_MACRO_INFO_DISABLE_CACHE__NO_UID__={macro} + }} + '''.format(c_debug=' '.join(c_debug_map), + c_debug_cl=' '.join(c_debug_map_cl), + yasm_debug=' '.join(yasm_debug_map), + c_debug_light=' '.join(c_debug_map_light), # build_root substitution only + c_debug_light_cl=' '.join(c_debug_map_light_cl), # build_root substitution only + yasm_debug_light=yasm_debug_map[0], # build_root substitution only + macro=' '.join(c_builtins))) + # TODO(somov): Check whether this specific architecture is needed. if self.target.arch == 'i386': append('CFLAGS', '-march=pentiumpro') @@ -1715,8 +1715,8 @@ class GnuCompiler(Compiler): '$CL_MACRO_INFO_DISABLE_CACHE__NO_UID__', '&& $COMPILER_TIME_TRACE_POSTPROCESS', ] - c_args_nodeps = [c if c != '$GCC_COMPILE_FLAGS' else '$EXTRA_C_FLAGS -c -o ${OUTFILE} ${SRC} ${pre=-I:INC}' for c in c_args if c not in ignore_c_args_no_deps] - + c_args_nodeps = [c if c != '$GCC_COMPILE_FLAGS' else '$EXTRA_C_FLAGS -c -o ${OUTFILE} ${SRC} ${pre=-I:INC}' for c in c_args if c not in ignore_c_args_no_deps] + emit('_SRC_C_NODEPS_CMD', ' '.join(c_args_nodeps)) emit('_SRC_CPP_CMD', ' '.join(cxx_args)) emit('_SRC_C_CMD', ' '.join(c_args)) @@ -1755,12 +1755,12 @@ class Linker(object): self.tc = tc self.build = build self.type = self._get_default_linker_type() - + def _get_default_linker_type(self): if not self.tc.is_from_arcadia: # External (e.g. system) toolchain: disable linker selection logic return None - + if self.build.target.is_android: # Android toolchain is NDK, LLD works on all supported platforms return Linker.LLD @@ -1810,8 +1810,8 @@ class LD(Linker): self.strip = self.tc.strip self.objcopy = self.tc.objcopy - self.musl = Setting('MUSL', convert=to_bool) - + self.musl = Setting('MUSL', convert=to_bool) + if self.ar is None: if target.is_apple: # Use libtool. cctools ar does not understand -M needed for archive merging @@ -1867,23 +1867,23 @@ class LD(Linker): else: self.ld_icf_flag = '' - if self.musl.value: - self.ld_flags.extend(['-Wl,--no-as-needed']) + if self.musl.value: + self.ld_flags.extend(['-Wl,--no-as-needed']) if self.tc.is_gcc: # MUSL_BFD: musl build uses --no-dynamic-linker linker flag # which gold doesn't know about. And we can only specify linker # type, not it's path as we do for Clang through linker selector. self.ld_flags.append('-fuse-ld=bfd') - elif target.is_linux: + elif target.is_linux: self.ld_flags.extend(['-ldl', '-lrt', '-Wl,--no-as-needed']) if self.tc.is_gcc: self.ld_flags.extend(('-Wl,-Bstatic', '-latomic', '-Wl,-Bdynamic')) - elif target.is_android: + elif target.is_android: self.ld_flags.extend(['-ldl', '-Wl,--no-as-needed']) if self.type == Linker.LLD and target.android_api < 29: # https://github.com/android/ndk/issues/1196 self.ld_flags.append('-Wl,--no-rosegment') - elif target.is_macos: + elif target.is_macos: self.ld_flags.append('-Wl,-no_deduplicate') if not self.tc.is_clang: self.ld_flags.append('-Wl,-no_compact_unwind') @@ -1989,10 +1989,10 @@ class LD(Linker): emit('C_LIBRARY_PATH') emit('C_SYSTEM_LIBRARIES_INTERCEPT') - if self.musl.value: - emit('C_SYSTEM_LIBRARIES', '-nostdlib') - else: - emit('C_SYSTEM_LIBRARIES', self.use_stdlib, self.thread_library, self.sys_lib, '-lc') + if self.musl.value: + emit('C_SYSTEM_LIBRARIES', '-nostdlib') + else: + emit('C_SYSTEM_LIBRARIES', self.use_stdlib, self.thread_library, self.sys_lib, '-lc') emit('START_WHOLE_ARCHIVE_VALUE', self.whole_archive) emit('END_WHOLE_ARCHIVE_VALUE', self.no_whole_archive) @@ -2142,7 +2142,7 @@ class LD(Linker): else: emit('LINK_DYN_LIB', '$GENERATE_MF && $GENERATE_VCS_C_INFO_NODEP && $REAL_LINK_DYN_LIB && $DWARF_COMMAND && $LINK_ADDITIONAL_SECTIONS_COMMAND') emit('LINK_EXEC_DYN_LIB', '$GENERATE_MF && $GENERATE_VCS_C_INFO_NODEP && $REAL_LINK_EXEC_DYN_LIB && $DWARF_COMMAND && $LINK_ADDITIONAL_SECTIONS_COMMAND') - emit('SWIG_DLL_JAR_CMD', '$GENERATE_MF && $GENERATE_VCS_C_INFO_NODEP && $REAL_SWIG_DLL_JAR_CMD && $DWARF_COMMAND') + emit('SWIG_DLL_JAR_CMD', '$GENERATE_MF && $GENERATE_VCS_C_INFO_NODEP && $REAL_SWIG_DLL_JAR_CMD && $DWARF_COMMAND') tail_link_lib = '$AUTO_INPUT ${requirements;hide:LIB_REQUIREMENTS} ${kv;hide:"p AR"} $TOOLCHAIN_ENV ${kv;hide:"pc light-red"} ${kv;hide:"show_out"}' if is_positive("TIDY"): @@ -2534,7 +2534,7 @@ class MSVCCompiler(MSVC, Compiler): if not self.tc.ide_msvs: def include_flag(path): - return '{flag}"{path}"'.format(path=path, flag='/I ' if not self.tc.use_clang else '-imsvc') + return '{flag}"{path}"'.format(path=path, flag='/I ' if not self.tc.use_clang else '-imsvc') for name in ('shared', 'ucrt', 'um', 'winrt'): flags.append(include_flag(os.path.join(self.tc.kit_includes, name))) |