diff options
author | snermolaev <snermolaev@yandex-team.ru> | 2022-02-10 16:45:53 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:53 +0300 |
commit | 7353a3fdea9c67c256980c00a2b3b67f09b23a27 (patch) | |
tree | 1a2c5ffcf89eb53ecd79dbc9bc0a195c27404d0c /build/plugins/gobuild.py | |
parent | 2015790ac9fcc04caab83fccc23ab2460310a797 (diff) | |
download | ydb-7353a3fdea9c67c256980c00a2b3b67f09b23a27.tar.gz |
Restoring authorship annotation for <snermolaev@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'build/plugins/gobuild.py')
-rw-r--r-- | build/plugins/gobuild.py | 548 |
1 files changed, 274 insertions, 274 deletions
diff --git a/build/plugins/gobuild.py b/build/plugins/gobuild.py index 3085eb8a5e..8df96ebc55 100644 --- a/build/plugins/gobuild.py +++ b/build/plugins/gobuild.py @@ -1,277 +1,277 @@ -import base64 -import itertools -import md5 -import os -from _common import rootrel_arc_src, tobuilddir -import ymake - - -runtime_cgo_path = os.path.join('runtime', 'cgo') -runtime_msan_path = os.path.join('runtime', 'msan') -runtime_race_path = os.path.join('runtime', 'race') -arc_project_prefix = 'a.yandex-team.ru/' -import_runtime_cgo_false = { - 'norace': (runtime_cgo_path, runtime_msan_path, runtime_race_path), - 'race': (runtime_cgo_path, runtime_msan_path), -} -import_syscall_false = { - 'norace': (runtime_cgo_path), - 'race': (runtime_cgo_path, runtime_race_path), -} - - -def get_import_path(unit): - # std_lib_prefix = unit.get('GO_STD_LIB_PREFIX') - # unit.get() doesn't evalutate the value of variable, so the line above doesn't really work - std_lib_prefix = unit.get('GOSTD') + '/' - arc_project_prefix = unit.get('GO_ARCADIA_PROJECT_PREFIX') - vendor_prefix = unit.get('GO_CONTRIB_PROJECT_PREFIX') - - module_path = rootrel_arc_src(unit.path(), unit) - assert len(module_path) > 0 - import_path = module_path.replace('\\', '/') - if import_path.startswith(std_lib_prefix): - import_path = import_path[len(std_lib_prefix):] - elif import_path.startswith(vendor_prefix): - import_path = import_path[len(vendor_prefix):] - else: - import_path = arc_project_prefix + import_path - assert len(import_path) > 0 - return import_path - - -def get_appended_values(unit, key): - value = [] - raw_value = unit.get(key) - if raw_value: - value = filter(lambda x: len(x) > 0, raw_value.split(' ')) - assert len(value) == 0 or value[0] == '$' + key - return value[1:] if len(value) > 0 else value - - -def compare_versions(version1, version2): - def last_index(version): - index = version.find('beta') - return len(version) if index < 0 else index - - v1 = tuple(x.zfill(8) for x in version1[:last_index(version1)].split('.')) - v2 = tuple(x.zfill(8) for x in version2[:last_index(version2)].split('.')) - if v1 == v2: - return 0 - return 1 if v1 < v2 else -1 - - -def need_compiling_runtime(import_path, gostd_version): - return import_path in ('runtime', 'reflect', 'syscall') or \ - import_path.startswith('runtime/internal/') or \ - compare_versions('1.17', gostd_version) >= 0 and import_path == 'internal/bytealg' - - -def go_package_name(unit): - name = unit.get('GO_PACKAGE_VALUE') - if not name: - name = unit.get('GO_TEST_IMPORT_PATH') - if name: - name = os.path.basename(os.path.normpath(name)) - elif unit.get('MODULE_TYPE') == 'PROGRAM': - name = 'main' - else: - name = unit.get('REALPRJNAME') - return name - - -def need_lint(path): - return not path.startswith('$S/vendor/') and not path.startswith('$S/contrib/') - - -def on_go_process_srcs(unit): - """ - _GO_PROCESS_SRCS() macro processes only 'CGO' files. All remaining *.go files - and other input files are currently processed by a link command of the - GO module (GO_LIBRARY, GO_PROGRAM) - """ - - srcs_files = get_appended_values(unit, '_GO_SRCS_VALUE') - - asm_files = [] - c_files = [] - cxx_files = [] - ev_files = [] - fbs_files = [] - go_files = [] - in_files = [] - proto_files = [] - s_files = [] - syso_files = [] - - classifed_files = { - '.c': c_files, - '.cc': cxx_files, - '.cpp': cxx_files, - '.cxx': cxx_files, - '.ev': ev_files, - '.fbs': fbs_files, - '.go': go_files, - '.in': in_files, - '.proto': proto_files, - '.s': asm_files, - '.syso': syso_files, - '.C': cxx_files, - '.S': s_files, - } - - # Classify files specifed in _GO_SRCS() macro by extension and process CGO_EXPORT keyword - # which can preceed C/C++ files only - is_cgo_export = False - for f in srcs_files: - _, ext = os.path.splitext(f) - ext_files = classifed_files.get(ext) - if ext_files is not None: - if is_cgo_export: - is_cgo_export = False - if ext in ('.c', '.cc', '.cpp', '.cxx', '.C'): - unit.oncopy_file_with_context([f, f, 'OUTPUT_INCLUDES', '${BINDIR}/_cgo_export.h']) - f = '${BINDIR}/' + f - else: - ymake.report_configure_error('Unmatched CGO_EXPORT keyword in SRCS() macro') - ext_files.append(f) - elif f == 'CGO_EXPORT': - is_cgo_export = True - else: - # FIXME(snermolaev): We can report an unsupported files for _GO_SRCS here - pass - if is_cgo_export: - ymake.report_configure_error('Unmatched CGO_EXPORT keyword in SRCS() macro') - - for f in go_files: - if f.endswith('_test.go'): - ymake.report_configure_error('file {} must be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f)) - go_test_files = get_appended_values(unit, '_GO_TEST_SRCS_VALUE') - go_xtest_files = get_appended_values(unit, '_GO_XTEST_SRCS_VALUE') - for f in go_test_files + go_xtest_files: - if not f.endswith('_test.go'): - ymake.report_configure_error('file {} should not be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f)) - - is_test_module = unit.enabled('GO_TEST_MODULE') - - # Add gofmt style checks - if unit.enabled('_GO_FMT_ADD_CHECK'): - resolved_go_files = [] - go_source_files = [] if is_test_module and unit.get(['GO_TEST_FOR_DIR']) else go_files - for path in itertools.chain(go_source_files, go_test_files, go_xtest_files): - if path.endswith('.go'): - resolved = unit.resolve_arc_path([path]) - if resolved != path and need_lint(resolved): - resolved_go_files.append(resolved) - if resolved_go_files: - basedirs = {} - for f in resolved_go_files: - basedir = os.path.dirname(f) - if basedir not in basedirs: - basedirs[basedir] = [] - basedirs[basedir].append(f) - for basedir in basedirs: - unit.onadd_check(['gofmt'] + basedirs[basedir]) - - # Go coverage instrumentation (NOTE! go_files list is modified here) - if is_test_module and unit.enabled('GO_TEST_COVER'): - cover_info = [] - - for f in go_files: - if f.endswith('_test.go'): - continue - cover_var = 'GoCover' + base64.b32encode(f).rstrip('=') - cover_file = unit.resolve_arc_path(f) - unit.on_go_gen_cover_go([cover_file, cover_var]) - if cover_file.startswith('$S/'): - cover_file = arc_project_prefix + cover_file[3:] - cover_info.append('{}:{}'.format(cover_var, cover_file)) - - # go_files should be empty now since the initial list shouldn't contain - # any non-go or go test file. The value of go_files list will be used later - # to update the value of _GO_SRCS_VALUE - go_files = [] - unit.set(['GO_COVER_INFO_VALUE', ' '.join(cover_info)]) - - # We have cleaned up the list of files from _GO_SRCS_VALUE var and we have to update - # the value since it is used in module command line - unit.set(['_GO_SRCS_VALUE', ' '.join(itertools.chain(go_files, asm_files, syso_files))]) - - unit_path = unit.path() - - # Add go vet check - if unit.enabled('_GO_VET_ADD_CHECK') and need_lint(unit_path): - vet_report_file_name = os.path.join(unit_path, '{}{}'.format(unit.filename(), unit.get('GO_VET_REPORT_EXT'))) - unit.onadd_check(["govet", '$(BUILD_ROOT)/' + tobuilddir(vet_report_file_name)[3:]]) - - for f in ev_files: - ev_proto_file = '{}.proto'.format(f) - unit.oncopy_file_with_context([f, ev_proto_file]) - proto_files.append(ev_proto_file) - - # Process .proto files - for f in proto_files: - unit.on_go_proto_cmd(f) - - # Process .fbs files - for f in fbs_files: - unit.on_go_flatc_cmd([f, go_package_name(unit)]) - - # Process .in files - for f in in_files: - unit.onsrc(f) - - # Generate .symabis for .s files (starting from 1.12 version) - if len(asm_files) > 0: - symabis_flags = [] - gostd_version = unit.get('GOSTD_VERSION') - if compare_versions('1.16', gostd_version) >= 0: - import_path = get_import_path(unit) - symabis_flags.extend(['FLAGS', '-p', import_path]) - if need_compiling_runtime(import_path, gostd_version): - symabis_flags.append('-compiling-runtime') - unit.on_go_compile_symabis(asm_files + symabis_flags) - - # Process cgo files - cgo_files = get_appended_values(unit, '_CGO_SRCS_VALUE') - - cgo_cflags = [] - if len(c_files) + len(cxx_files) + len(s_files) + len(cgo_files) > 0: - if is_test_module: - go_test_for_dir = unit.get('GO_TEST_FOR_DIR') - if go_test_for_dir and go_test_for_dir.startswith('$S/'): - unit.onaddincl(['FOR', 'c', go_test_for_dir[3:]]) - unit.onaddincl(['FOR', 'c', unit.get('MODDIR')]) - cgo_cflags = get_appended_values(unit, 'CGO_CFLAGS_VALUE') - - for f in itertools.chain(c_files, cxx_files, s_files): - unit.onsrc([f] + cgo_cflags) - - if len(cgo_files) > 0: - if not unit.enabled('CGO_ENABLED'): - ymake.report_configure_error('trying to build with CGO (CGO_SRCS is non-empty) when CGO is disabled') - import_path = get_import_path(unit) - if import_path != runtime_cgo_path: - go_std_root = unit.get('GOSTD') - unit.onpeerdir(os.path.join(go_std_root, runtime_cgo_path)) - race_mode = 'race' if unit.enabled('RACE') else 'norace' - import_runtime_cgo = 'false' if import_path in import_runtime_cgo_false[race_mode] else 'true' - import_syscall = 'false' if import_path in import_syscall_false[race_mode] else 'true' - args = [import_path] + cgo_files + ['FLAGS', '-import_runtime_cgo=' + import_runtime_cgo, '-import_syscall=' + import_syscall] +import base64 +import itertools +import md5 +import os +from _common import rootrel_arc_src, tobuilddir +import ymake + + +runtime_cgo_path = os.path.join('runtime', 'cgo') +runtime_msan_path = os.path.join('runtime', 'msan') +runtime_race_path = os.path.join('runtime', 'race') +arc_project_prefix = 'a.yandex-team.ru/' +import_runtime_cgo_false = { + 'norace': (runtime_cgo_path, runtime_msan_path, runtime_race_path), + 'race': (runtime_cgo_path, runtime_msan_path), +} +import_syscall_false = { + 'norace': (runtime_cgo_path), + 'race': (runtime_cgo_path, runtime_race_path), +} + + +def get_import_path(unit): + # std_lib_prefix = unit.get('GO_STD_LIB_PREFIX') + # unit.get() doesn't evalutate the value of variable, so the line above doesn't really work + std_lib_prefix = unit.get('GOSTD') + '/' + arc_project_prefix = unit.get('GO_ARCADIA_PROJECT_PREFIX') + vendor_prefix = unit.get('GO_CONTRIB_PROJECT_PREFIX') + + module_path = rootrel_arc_src(unit.path(), unit) + assert len(module_path) > 0 + import_path = module_path.replace('\\', '/') + if import_path.startswith(std_lib_prefix): + import_path = import_path[len(std_lib_prefix):] + elif import_path.startswith(vendor_prefix): + import_path = import_path[len(vendor_prefix):] + else: + import_path = arc_project_prefix + import_path + assert len(import_path) > 0 + return import_path + + +def get_appended_values(unit, key): + value = [] + raw_value = unit.get(key) + if raw_value: + value = filter(lambda x: len(x) > 0, raw_value.split(' ')) + assert len(value) == 0 or value[0] == '$' + key + return value[1:] if len(value) > 0 else value + + +def compare_versions(version1, version2): + def last_index(version): + index = version.find('beta') + return len(version) if index < 0 else index + + v1 = tuple(x.zfill(8) for x in version1[:last_index(version1)].split('.')) + v2 = tuple(x.zfill(8) for x in version2[:last_index(version2)].split('.')) + if v1 == v2: + return 0 + return 1 if v1 < v2 else -1 + + +def need_compiling_runtime(import_path, gostd_version): + return import_path in ('runtime', 'reflect', 'syscall') or \ + import_path.startswith('runtime/internal/') or \ + compare_versions('1.17', gostd_version) >= 0 and import_path == 'internal/bytealg' + + +def go_package_name(unit): + name = unit.get('GO_PACKAGE_VALUE') + if not name: + name = unit.get('GO_TEST_IMPORT_PATH') + if name: + name = os.path.basename(os.path.normpath(name)) + elif unit.get('MODULE_TYPE') == 'PROGRAM': + name = 'main' + else: + name = unit.get('REALPRJNAME') + return name + + +def need_lint(path): + return not path.startswith('$S/vendor/') and not path.startswith('$S/contrib/') + + +def on_go_process_srcs(unit): + """ + _GO_PROCESS_SRCS() macro processes only 'CGO' files. All remaining *.go files + and other input files are currently processed by a link command of the + GO module (GO_LIBRARY, GO_PROGRAM) + """ + + srcs_files = get_appended_values(unit, '_GO_SRCS_VALUE') + + asm_files = [] + c_files = [] + cxx_files = [] + ev_files = [] + fbs_files = [] + go_files = [] + in_files = [] + proto_files = [] + s_files = [] + syso_files = [] + + classifed_files = { + '.c': c_files, + '.cc': cxx_files, + '.cpp': cxx_files, + '.cxx': cxx_files, + '.ev': ev_files, + '.fbs': fbs_files, + '.go': go_files, + '.in': in_files, + '.proto': proto_files, + '.s': asm_files, + '.syso': syso_files, + '.C': cxx_files, + '.S': s_files, + } + + # Classify files specifed in _GO_SRCS() macro by extension and process CGO_EXPORT keyword + # which can preceed C/C++ files only + is_cgo_export = False + for f in srcs_files: + _, ext = os.path.splitext(f) + ext_files = classifed_files.get(ext) + if ext_files is not None: + if is_cgo_export: + is_cgo_export = False + if ext in ('.c', '.cc', '.cpp', '.cxx', '.C'): + unit.oncopy_file_with_context([f, f, 'OUTPUT_INCLUDES', '${BINDIR}/_cgo_export.h']) + f = '${BINDIR}/' + f + else: + ymake.report_configure_error('Unmatched CGO_EXPORT keyword in SRCS() macro') + ext_files.append(f) + elif f == 'CGO_EXPORT': + is_cgo_export = True + else: + # FIXME(snermolaev): We can report an unsupported files for _GO_SRCS here + pass + if is_cgo_export: + ymake.report_configure_error('Unmatched CGO_EXPORT keyword in SRCS() macro') + + for f in go_files: + if f.endswith('_test.go'): + ymake.report_configure_error('file {} must be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f)) + go_test_files = get_appended_values(unit, '_GO_TEST_SRCS_VALUE') + go_xtest_files = get_appended_values(unit, '_GO_XTEST_SRCS_VALUE') + for f in go_test_files + go_xtest_files: + if not f.endswith('_test.go'): + ymake.report_configure_error('file {} should not be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f)) + + is_test_module = unit.enabled('GO_TEST_MODULE') + + # Add gofmt style checks + if unit.enabled('_GO_FMT_ADD_CHECK'): + resolved_go_files = [] + go_source_files = [] if is_test_module and unit.get(['GO_TEST_FOR_DIR']) else go_files + for path in itertools.chain(go_source_files, go_test_files, go_xtest_files): + if path.endswith('.go'): + resolved = unit.resolve_arc_path([path]) + if resolved != path and need_lint(resolved): + resolved_go_files.append(resolved) + if resolved_go_files: + basedirs = {} + for f in resolved_go_files: + basedir = os.path.dirname(f) + if basedir not in basedirs: + basedirs[basedir] = [] + basedirs[basedir].append(f) + for basedir in basedirs: + unit.onadd_check(['gofmt'] + basedirs[basedir]) + + # Go coverage instrumentation (NOTE! go_files list is modified here) + if is_test_module and unit.enabled('GO_TEST_COVER'): + cover_info = [] + + for f in go_files: + if f.endswith('_test.go'): + continue + cover_var = 'GoCover' + base64.b32encode(f).rstrip('=') + cover_file = unit.resolve_arc_path(f) + unit.on_go_gen_cover_go([cover_file, cover_var]) + if cover_file.startswith('$S/'): + cover_file = arc_project_prefix + cover_file[3:] + cover_info.append('{}:{}'.format(cover_var, cover_file)) + + # go_files should be empty now since the initial list shouldn't contain + # any non-go or go test file. The value of go_files list will be used later + # to update the value of _GO_SRCS_VALUE + go_files = [] + unit.set(['GO_COVER_INFO_VALUE', ' '.join(cover_info)]) + + # We have cleaned up the list of files from _GO_SRCS_VALUE var and we have to update + # the value since it is used in module command line + unit.set(['_GO_SRCS_VALUE', ' '.join(itertools.chain(go_files, asm_files, syso_files))]) + + unit_path = unit.path() + + # Add go vet check + if unit.enabled('_GO_VET_ADD_CHECK') and need_lint(unit_path): + vet_report_file_name = os.path.join(unit_path, '{}{}'.format(unit.filename(), unit.get('GO_VET_REPORT_EXT'))) + unit.onadd_check(["govet", '$(BUILD_ROOT)/' + tobuilddir(vet_report_file_name)[3:]]) + + for f in ev_files: + ev_proto_file = '{}.proto'.format(f) + unit.oncopy_file_with_context([f, ev_proto_file]) + proto_files.append(ev_proto_file) + + # Process .proto files + for f in proto_files: + unit.on_go_proto_cmd(f) + + # Process .fbs files + for f in fbs_files: + unit.on_go_flatc_cmd([f, go_package_name(unit)]) + + # Process .in files + for f in in_files: + unit.onsrc(f) + + # Generate .symabis for .s files (starting from 1.12 version) + if len(asm_files) > 0: + symabis_flags = [] + gostd_version = unit.get('GOSTD_VERSION') + if compare_versions('1.16', gostd_version) >= 0: + import_path = get_import_path(unit) + symabis_flags.extend(['FLAGS', '-p', import_path]) + if need_compiling_runtime(import_path, gostd_version): + symabis_flags.append('-compiling-runtime') + unit.on_go_compile_symabis(asm_files + symabis_flags) + + # Process cgo files + cgo_files = get_appended_values(unit, '_CGO_SRCS_VALUE') + + cgo_cflags = [] + if len(c_files) + len(cxx_files) + len(s_files) + len(cgo_files) > 0: + if is_test_module: + go_test_for_dir = unit.get('GO_TEST_FOR_DIR') + if go_test_for_dir and go_test_for_dir.startswith('$S/'): + unit.onaddincl(['FOR', 'c', go_test_for_dir[3:]]) + unit.onaddincl(['FOR', 'c', unit.get('MODDIR')]) + cgo_cflags = get_appended_values(unit, 'CGO_CFLAGS_VALUE') + + for f in itertools.chain(c_files, cxx_files, s_files): + unit.onsrc([f] + cgo_cflags) + + if len(cgo_files) > 0: + if not unit.enabled('CGO_ENABLED'): + ymake.report_configure_error('trying to build with CGO (CGO_SRCS is non-empty) when CGO is disabled') + import_path = get_import_path(unit) + if import_path != runtime_cgo_path: + go_std_root = unit.get('GOSTD') + unit.onpeerdir(os.path.join(go_std_root, runtime_cgo_path)) + race_mode = 'race' if unit.enabled('RACE') else 'norace' + import_runtime_cgo = 'false' if import_path in import_runtime_cgo_false[race_mode] else 'true' + import_syscall = 'false' if import_path in import_syscall_false[race_mode] else 'true' + args = [import_path] + cgo_files + ['FLAGS', '-import_runtime_cgo=' + import_runtime_cgo, '-import_syscall=' + import_syscall] unit.on_go_compile_cgo1(args) - cgo2_cflags = get_appended_values(unit, 'CGO2_CFLAGS_VALUE') - for f in cgo_files: - if f.endswith('.go'): - unit.onsrc([f[:-2] + 'cgo2.c'] + cgo_cflags + cgo2_cflags) - else: - ymake.report_configure_error('file {} should not be listed in CGO_SRCS() macros'.format(f)) + cgo2_cflags = get_appended_values(unit, 'CGO2_CFLAGS_VALUE') + for f in cgo_files: + if f.endswith('.go'): + unit.onsrc([f[:-2] + 'cgo2.c'] + cgo_cflags + cgo2_cflags) + else: + ymake.report_configure_error('file {} should not be listed in CGO_SRCS() macros'.format(f)) args = [go_package_name(unit)] + cgo_files - if len(c_files) > 0: - args += ['C_FILES'] + c_files - if len(s_files) > 0: - args += ['S_FILES'] + s_files - if len(syso_files) > 0: - args += ['OBJ_FILES'] + syso_files + if len(c_files) > 0: + args += ['C_FILES'] + c_files + if len(s_files) > 0: + args += ['S_FILES'] + s_files + if len(syso_files) > 0: + args += ['OBJ_FILES'] + syso_files unit.on_go_compile_cgo2(args) @@ -279,8 +279,8 @@ def on_go_resource(unit, *args): args = list(args) files = args[::2] keys = args[1::2] - suffix_md5 = md5.new('@'.join(args)).hexdigest() - resource_go = os.path.join("resource.{}.res.go".format(suffix_md5)) + suffix_md5 = md5.new('@'.join(args)).hexdigest() + resource_go = os.path.join("resource.{}.res.go".format(suffix_md5)) unit.onpeerdir(["library/go/core/resource"]) |