diff options
author | neksard <neksard@yandex-team.ru> | 2022-02-10 16:45:23 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:23 +0300 |
commit | 8f7cf138264e0caa318144bf8a2c950e0b0a8593 (patch) | |
tree | 83bf5c8c8047c42d8475e6095df90ccdc3d1b57f /build | |
parent | d3a398281c6fd1d3672036cb2d63f842d2cb28c5 (diff) | |
download | ydb-8f7cf138264e0caa318144bf8a2c950e0b0a8593.tar.gz |
Restoring authorship annotation for <neksard@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'build')
-rw-r--r-- | build/platform/cuda/ya.make | 4 | ||||
-rw-r--r-- | build/platform/msvc/ya.make | 4 | ||||
-rw-r--r-- | build/platform/perl/ya.make | 16 | ||||
-rw-r--r-- | build/platform/python/readme.md | 158 | ||||
-rw-r--r-- | build/platform/python/ya.make | 42 | ||||
-rw-r--r-- | build/platform/wine/ya.make | 14 | ||||
-rw-r--r-- | build/plugins/ytest.py | 10 | ||||
-rw-r--r-- | build/scripts/compile_cuda.py | 26 | ||||
-rw-r--r-- | build/scripts/coverage-info.py | 10 | ||||
-rw-r--r-- | build/scripts/fetch_from_external.py | 16 | ||||
-rw-r--r-- | build/scripts/link_dyn_lib.py | 262 | ||||
-rw-r--r-- | build/scripts/run_msvc_wine.py | 74 | ||||
-rw-r--r-- | build/scripts/ya.make | 10 | ||||
-rw-r--r-- | build/sysincl/misc.yml | 4 | ||||
-rw-r--r-- | build/ya.conf.json | 42 | ||||
-rw-r--r-- | build/ymake.core.conf | 120 | ||||
-rwxr-xr-x | build/ymake_conf.py | 80 |
17 files changed, 446 insertions, 446 deletions
diff --git a/build/platform/cuda/ya.make b/build/platform/cuda/ya.make index 3615f682cd..493ff6d36e 100644 --- a/build/platform/cuda/ya.make +++ b/build/platform/cuda/ya.make @@ -18,7 +18,7 @@ IF (USE_ARCADIA_CUDA) ELSEIF (CUDA_VERSION == "11.2") DECLARE_EXTERNAL_RESOURCE(CUDA sbr:2073566375) # CUDA Toolkit 11.2.2 for Linux x86-64 ELSEIF (CUDA_VERSION == "11.1") - DECLARE_EXTERNAL_RESOURCE(CUDA sbr:1882836946) # CUDA Toolkit 11.1.1 for Linux x86-64 + DECLARE_EXTERNAL_RESOURCE(CUDA sbr:1882836946) # CUDA Toolkit 11.1.1 for Linux x86-64 ELSEIF (CUDA_VERSION == "11.0") DECLARE_EXTERNAL_RESOURCE(CUDA sbr:1647896014) # CUDA Toolkit 11.0.2 for Linux x86-64 ELSEIF (CUDA_VERSION == "10.1") @@ -65,7 +65,7 @@ IF (USE_ARCADIA_CUDA) IF (CUDA_VERSION == "11.3") DECLARE_EXTERNAL_RESOURCE(CUDA sbr:2215101513) # CUDA Toolkit 11.3.1 for Windows x86-64 ELSEIF (CUDA_VERSION == "11.1") - DECLARE_EXTERNAL_RESOURCE(CUDA sbr:1896564605) # CUDA Toolkit 11.1.1 for Windows x86-64 + DECLARE_EXTERNAL_RESOURCE(CUDA sbr:1896564605) # CUDA Toolkit 11.1.1 for Windows x86-64 ELSEIF (CUDA_VERSION == "10.1") DECLARE_EXTERNAL_RESOURCE(CUDA sbr:978734165) # CUDA Toolkit 10.1.168 for Windows x86-64 ELSE() diff --git a/build/platform/msvc/ya.make b/build/platform/msvc/ya.make index ccb602c280..c4887518c9 100644 --- a/build/platform/msvc/ya.make +++ b/build/platform/msvc/ya.make @@ -6,8 +6,8 @@ IF (WINDOWS_KITS_VERSION == "10.0.10586.0") DECLARE_EXTERNAL_RESOURCE(WINDOWS_KITS sbr:544779014) ELSEIF (WINDOWS_KITS_VERSION == "10.0.16299.0") DECLARE_EXTERNAL_RESOURCE(WINDOWS_KITS sbr:1379398385) -ELSEIF (WINDOWS_KITS_VERSION == "10.0.18362.0") - DECLARE_EXTERNAL_RESOURCE(WINDOWS_KITS sbr:1939557911) +ELSEIF (WINDOWS_KITS_VERSION == "10.0.18362.0") + DECLARE_EXTERNAL_RESOURCE(WINDOWS_KITS sbr:1939557911) ELSE() MESSAGE(FATAL_ERROR "We have no Windows Kits version ${WINDOWS_KITS_VERSION}") ENDIF() diff --git a/build/platform/perl/ya.make b/build/platform/perl/ya.make index cc50c05c26..e9b3ab30d1 100644 --- a/build/platform/perl/ya.make +++ b/build/platform/perl/ya.make @@ -1,10 +1,10 @@ LIBRARY() - -OWNER(somov) - + +OWNER(somov) + NO_PLATFORM() -IF (USE_PERL_SYSTEM) +IF (USE_PERL_SYSTEM) IF (PERL_SDK == "ubuntu-12") PEERDIR(build/platform/perl/5.14) ELSEIF (PERL_SDK == "ubuntu-14") @@ -19,10 +19,10 @@ IF (USE_PERL_SYSTEM) MESSAGE(FATAL_ERROR "Building against system perl is not supported on ${PERL_SDK}") ENDIF() -ELSE() +ELSE() MESSAGE(FATAL_ERROR "There is no perl ready for static linkage. Try using the system one.") -ENDIF() - -END() +ENDIF() + +END() diff --git a/build/platform/python/readme.md b/build/platform/python/readme.md index 1eaafad35b..5e74dcb6f6 100644 --- a/build/platform/python/readme.md +++ b/build/platform/python/readme.md @@ -1,83 +1,83 @@ -# Инструкция по добавлению нового бандла с системным питоном - -## Майним бандлы системного питона -Бандлы системного питон майнятся для трех платформ: linux, darwin, windows. -Подставляем под PYTHON_VERSION - версию нужного питона -### Linux - -1. Устанавливаем систему с версией ubuntu, из которой планируется брать системный питон. Здесь есть два варианта - 1. Если нужно собрать системный питон, который будет запускать тесты на дистбилде, то нужно использовать ubuntu такой же версии, что и на дистбилде. - Тут стоит учитывать, что на дистбилде может быть достаточно старая версия ubuntu, на котором не будет нужной версии питона. - 2. Выбрать ту версию ubuntu, в которой есть нужный питон -2. `mkdir -p ~/work/packages` -3. `cd ~/work/packages` -4. майним deb-пакеты питона - 1. Майним системный питон для запуска на дистбилде: - - apt-get download $(apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances python{PYTHON_VERSION}-dev python{|3}-pkg-resources python{|3}-setuptools | grep "^\w" | sort -u) - rm libc6* libc-* - - 2. Майним системный питон для сборки сошек: - - apt download python{PYTHON_VERSION} python{PYTHON_VERSION}-dev python{PYTHON_VERSION}-minimal libpython{PYTHON_VERSION} libpython{PYTHON_VERSION}-dev libpython{PYTHON_VERSION}-stdlib libpython{PYTHON_VERSION}-minimal - -5. `cd ..` -6. `for path in $(ls packages); do ar -xf packages/$path; tar -xf data.tar.xz; done;` -7. `mv usr python` -8. `tar -czf python{PYTHON_VERSION}_linux.tar.gz python` +# Инструкция по добавлению нового бандла с системным питоном + +## Майним бандлы системного питона +Бандлы системного питон майнятся для трех платформ: linux, darwin, windows. +Подставляем под PYTHON_VERSION - версию нужного питона +### Linux + +1. Устанавливаем систему с версией ubuntu, из которой планируется брать системный питон. Здесь есть два варианта + 1. Если нужно собрать системный питон, который будет запускать тесты на дистбилде, то нужно использовать ubuntu такой же версии, что и на дистбилде. + Тут стоит учитывать, что на дистбилде может быть достаточно старая версия ubuntu, на котором не будет нужной версии питона. + 2. Выбрать ту версию ubuntu, в которой есть нужный питон +2. `mkdir -p ~/work/packages` +3. `cd ~/work/packages` +4. майним deb-пакеты питона + 1. Майним системный питон для запуска на дистбилде: + + apt-get download $(apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances python{PYTHON_VERSION}-dev python{|3}-pkg-resources python{|3}-setuptools | grep "^\w" | sort -u) + rm libc6* libc-* + + 2. Майним системный питон для сборки сошек: + + apt download python{PYTHON_VERSION} python{PYTHON_VERSION}-dev python{PYTHON_VERSION}-minimal libpython{PYTHON_VERSION} libpython{PYTHON_VERSION}-dev libpython{PYTHON_VERSION}-stdlib libpython{PYTHON_VERSION}-minimal + +5. `cd ..` +6. `for path in $(ls packages); do ar -xf packages/$path; tar -xf data.tar.xz; done;` +7. `mv usr python` +8. `tar -czf python{PYTHON_VERSION}_linux.tar.gz python` 9. `ya upload python{PYTHON_VERSION}_linux.tar.gz -d "Ubuntu {UBUNTU_VERSION} x86_64 python{PYTHON_VERSION} installation" --do-not-remove` - -UBUNTU_VERSION - версия ubuntu, на которой майнился системный питон - -### Darwin -1. Находим macbook. -2. Все установленные питоны лежат в `/Library/Frameworks/Python.framework/Versions` -3. Копируем `/Library/Frameworks/Python.framework` в директорию с именем `python` -4. Чистим `python/Python.framework/Versions/` от ненужных питонов -5. Проверяем, что симлинки указывают в правильные места - 1. `python/Python.framework/Versions/Current -> {PYTHON_VERSION}` - 2. `python/Python.framework/Headers -> Versions/Current/Headers` - 3. `python/Python.framework/Python -> Versions/Current/Python` - 4. `python/Python.framework/Resources -> Versions/Current/Resources` -6. `tar -czf python{PYTHON_VERSION}_darwin.tar.gz python` + +UBUNTU_VERSION - версия ubuntu, на которой майнился системный питон + +### Darwin +1. Находим macbook. +2. Все установленные питоны лежат в `/Library/Frameworks/Python.framework/Versions` +3. Копируем `/Library/Frameworks/Python.framework` в директорию с именем `python` +4. Чистим `python/Python.framework/Versions/` от ненужных питонов +5. Проверяем, что симлинки указывают в правильные места + 1. `python/Python.framework/Versions/Current -> {PYTHON_VERSION}` + 2. `python/Python.framework/Headers -> Versions/Current/Headers` + 3. `python/Python.framework/Python -> Versions/Current/Python` + 4. `python/Python.framework/Resources -> Versions/Current/Resources` +6. `tar -czf python{PYTHON_VERSION}_darwin.tar.gz python` 7. `ya upload python{PYTHON_VERSION}_darwin.tar.gz -d "Darwin x86_64 python{PYTHON_VERSION} installation" --do-not-remove` - -Если нужного питона нет в системе, его нужно установить из `python.org`, его установку можно найти в стандартном месте. - -Если нужен питон из `brew`, его установку можно найти в `/usr/local/Cellar/python*/{python_version}/Frameworks/`, - а дальше следовать стандартной инструкции - -### Windows -1. Находим машинку с windows -2. Устанавливаем нужную версию питона из `python.org` -3. Копируем содержимое установки питона в директорию `python` -4. Пакуем директорию `python` в `python{PYTHON_VERSION}_windows.tar.gz` + +Если нужного питона нет в системе, его нужно установить из `python.org`, его установку можно найти в стандартном месте. + +Если нужен питон из `brew`, его установку можно найти в `/usr/local/Cellar/python*/{python_version}/Frameworks/`, + а дальше следовать стандартной инструкции + +### Windows +1. Находим машинку с windows +2. Устанавливаем нужную версию питона из `python.org` +3. Копируем содержимое установки питона в директорию `python` +4. Пакуем директорию `python` в `python{PYTHON_VERSION}_windows.tar.gz` 5. `ya upload python{PYTHON_VERSION}_windows.tar.gz -d "Windows x86_64 python{PYTHON_VERSION} installation" --do-not-remove` - -## Добавляем бандлы системного питона в сборку - -1. Конфигурация бандлов системных питонов находится здесь [build/platform/python](https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python) -2. Добавляем сендбокс ресурсы собранных бандлов в файл [resources.inc](https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc) - - SET(PYTHON38_LINUX sbr:1211259884) - -3. Добавляем служебные переменные `_SYSTEM_PYTHON*, PY_VERSION, PY_FRAMEWORK_VERSION` для системного питона, если их еще нет, -в [ymake.core.conf](https://a.yandex-team.ru/arc/trunk/arcadia/build/ymake.core.conf?rev=7640792#L380) по аналогии. - - "3.8" ? { - _SYSTEM_PYTHON38=yes - PY_VERSION=3.8 - PY_FRAMEWORK_VERSION=3.8 - } - -4. Добавляем ресурс в [build/platform/python/ya.make](https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/ya.make) - - DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON38_LINUX}) - -## Проверяем сборку + +## Добавляем бандлы системного питона в сборку + +1. Конфигурация бандлов системных питонов находится здесь [build/platform/python](https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python) +2. Добавляем сендбокс ресурсы собранных бандлов в файл [resources.inc](https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/resources.inc) + + SET(PYTHON38_LINUX sbr:1211259884) + +3. Добавляем служебные переменные `_SYSTEM_PYTHON*, PY_VERSION, PY_FRAMEWORK_VERSION` для системного питона, если их еще нет, +в [ymake.core.conf](https://a.yandex-team.ru/arc/trunk/arcadia/build/ymake.core.conf?rev=7640792#L380) по аналогии. + + "3.8" ? { + _SYSTEM_PYTHON38=yes + PY_VERSION=3.8 + PY_FRAMEWORK_VERSION=3.8 + } + +4. Добавляем ресурс в [build/platform/python/ya.make](https://a.yandex-team.ru/arc/trunk/arcadia/build/platform/python/ya.make) + + DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON38_LINUX}) + +## Проверяем сборку 1. Создаем тривиальный PY2MODULE с использование `c api` положенного питона, или находим подходящий в репозитории -2. Собираем его: - 1. linux `ya make -DUSE_SYSTEM_PYTHON=3.8 --target-platform linux` - 2. darwin `ya make -DUSE_SYSTEM_PYTHON=3.8 --target-platform darwin` - 3. windows `ya make -DUSE_SYSTEM_PYTHON=3.8 --target-platform win` -3. Проверяем, что получившиеся модули импортятся в питонах на соответсвующих системах +2. Собираем его: + 1. linux `ya make -DUSE_SYSTEM_PYTHON=3.8 --target-platform linux` + 2. darwin `ya make -DUSE_SYSTEM_PYTHON=3.8 --target-platform darwin` + 3. windows `ya make -DUSE_SYSTEM_PYTHON=3.8 --target-platform win` +3. Проверяем, что получившиеся модули импортятся в питонах на соответсвующих системах diff --git a/build/platform/python/ya.make b/build/platform/python/ya.make index 247e65f4c9..2a8854baed 100644 --- a/build/platform/python/ya.make +++ b/build/platform/python/ya.make @@ -1,14 +1,14 @@ -RESOURCES_LIBRARY() - +RESOURCES_LIBRARY() + OWNER( g:contrib g:yatool ) - + INCLUDE(resources.inc) IF (USE_SYSTEM_PYTHON) - IF (OS_LINUX) + IF (OS_LINUX) IF (ARCH_ARM64 OR ARCH_AARCH64) IF (_SYSTEM_PYTHON27) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON27_LINUX_ARM64}) @@ -37,12 +37,12 @@ IF (USE_SYSTEM_PYTHON) ENDIF() ELSE() ENABLE(PYTHON_NOT_FOUND) - ENDIF() - CFLAGS( + ENDIF() + CFLAGS( GLOBAL "-isystem$EXTERNAL_PYTHON_RESOURCE_GLOBAL/python/include" GLOBAL "-isystem$EXTERNAL_PYTHON_RESOURCE_GLOBAL/python/include/python${PY_VERSION}" ) - ELSEIF (OS_DARWIN) + ELSEIF (OS_DARWIN) IF (PYTHON_DISTR == "macos-brew") IF (ARCH_X86_64) IF (_SYSTEM_PYTHON27) @@ -54,8 +54,8 @@ IF (USE_SYSTEM_PYTHON) ENDIF() ELSE() ENABLE(PYTHON_NOT_FOUND) - ENDIF() - ELSE() + ENDIF() + ELSE() IF (ARCH_X86_64) IF (_SYSTEM_PYTHON27) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON27_DARWIN}) @@ -93,17 +93,17 @@ IF (USE_SYSTEM_PYTHON) CFLAGS(GLOBAL "-isystem$EXTERNAL_PYTHON_RESOURCE_GLOBAL/python/Python.framework/Versions/${PY_FRAMEWORK_VERSION}/include/python${PY_VERSION}") ELSEIF (OS_WINDOWS) IF (ARCH_X86_64) - IF (_SYSTEM_PYTHON27) + IF (_SYSTEM_PYTHON27) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON27_WINDOWS}) - ELSEIF (_SYSTEM_PYTHON34) + ELSEIF (_SYSTEM_PYTHON34) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON34_WINDOWS}) - ELSEIF (_SYSTEM_PYTHON35) + ELSEIF (_SYSTEM_PYTHON35) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON35_WINDOWS}) - ELSEIF (_SYSTEM_PYTHON36) + ELSEIF (_SYSTEM_PYTHON36) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON36_WINDOWS}) - ELSEIF (_SYSTEM_PYTHON37) + ELSEIF (_SYSTEM_PYTHON37) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON37_WINDOWS}) - ELSEIF (_SYSTEM_PYTHON38) + ELSEIF (_SYSTEM_PYTHON38) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON38_WINDOWS}) ELSEIF (_SYSTEM_PYTHON39) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON39_WINDOWS}) @@ -111,11 +111,11 @@ IF (USE_SYSTEM_PYTHON) DECLARE_EXTERNAL_RESOURCE(EXTERNAL_PYTHON ${PYTHON310_WINDOWS}) ELSE() ENABLE(PYTHON_NOT_FOUND) - ENDIF() + ENDIF() ELSE() ENABLE(PYTHON_NOT_FOUND) - ENDIF() - CFLAGS(GLOBAL "/I$EXTERNAL_PYTHON_RESOURCE_GLOBAL/python/include") + ENDIF() + CFLAGS(GLOBAL "/I$EXTERNAL_PYTHON_RESOURCE_GLOBAL/python/include") LDFLAGS("/LIBPATH:$EXTERNAL_PYTHON_RESOURCE_GLOBAL/python/libs") ENDIF() @@ -127,9 +127,9 @@ ELSEIF (NOT USE_ARCADIA_PYTHON) IF (OS_WINDOWS) LDFLAGS($PYTHON_LDFLAGS $PYTHON_LIBRARIES) ENDIF() -ENDIF() - -END() +ENDIF() + +END() RECURSE_FOR_TESTS( tests diff --git a/build/platform/wine/ya.make b/build/platform/wine/ya.make index 7d74cb279b..511be37962 100644 --- a/build/platform/wine/ya.make +++ b/build/platform/wine/ya.make @@ -1,10 +1,10 @@ -RESOURCES_LIBRARY() - -OWNER(neksard) - +RESOURCES_LIBRARY() + +OWNER(neksard) + # do not forget to update resources in ya.conf.json -DECLARE_EXTERNAL_RESOURCE(WINE_TOOL sbr:1093314933) +DECLARE_EXTERNAL_RESOURCE(WINE_TOOL sbr:1093314933) DECLARE_EXTERNAL_RESOURCE(WINE32_TOOL sbr:2264052281) - + SET(NEED_PLATFORM_PEERDIRS no) -END() +END() diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index 8970837f0f..4fc6b1d590 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -1038,11 +1038,11 @@ def onsetup_exectest(unit, *args): unit.ondepends('contrib/tools/python') unit.set(["TEST_BLOB_DATA", base64.b64encode(command)]) add_test_to_dart(unit, "exectest", binary_path=os.path.join(unit.path(), unit.filename()).replace(".pkg", "")) - - -def onsetup_run_python(unit): - if unit.get("USE_ARCADIA_PYTHON") == "yes": - unit.ondepends('contrib/tools/python') + + +def onsetup_run_python(unit): + if unit.get("USE_ARCADIA_PYTHON") == "yes": + unit.ondepends('contrib/tools/python') def get_canonical_test_resources(unit): diff --git a/build/scripts/compile_cuda.py b/build/scripts/compile_cuda.py index c0bec50b2a..e393b7ecf1 100644 --- a/build/scripts/compile_cuda.py +++ b/build/scripts/compile_cuda.py @@ -1,18 +1,18 @@ -import sys -import subprocess +import sys +import subprocess import os import collections import re import tempfile - - -def is_clang(command): - for word in command: - if '--compiler-bindir' in word and 'clang' in word: - return True - - return False - + + +def is_clang(command): + for word in command: + if '--compiler-bindir' in word and 'clang' in word: + return True + + return False + def main(): try: @@ -25,7 +25,7 @@ def main(): mtime0 = sys.argv[1] command = sys.argv[2: spl] cflags = sys.argv[spl + 1:] - + dump_args = False if '--y_dump_args' in command: command.remove('--y_dump_args') @@ -35,7 +35,7 @@ def main(): if not os.path.exists(executable): print >> sys.stderr, '{} not found'.format(executable) sys.exit(1) - + if is_clang(command): # nvcc concatenates the sources for clang, and clang reports unused # things from .h files as if they they were defined in a .cpp file. diff --git a/build/scripts/coverage-info.py b/build/scripts/coverage-info.py index 94491d9256..7dc2691d23 100644 --- a/build/scripts/coverage-info.py +++ b/build/scripts/coverage-info.py @@ -237,10 +237,10 @@ def main(source_root, output, gcno_archive, gcda_archive, gcov_tool, prefix_filt init_all_coverage_files(gcno_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info, prefix_filter, exclude_files) process_all_coverage_files(gcda_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info) - if coverage_report_path: - output_dir = coverage_report_path - else: - output_dir = output + '.dir' + if coverage_report_path: + output_dir = coverage_report_path + else: + output_dir = output + '.dir' if not os.path.exists(output_dir): os.makedirs(output_dir) @@ -274,7 +274,7 @@ if __name__ == '__main__': parser.add_argument('--prefix-filter', action='store') parser.add_argument('--exclude-regexp', action='store') parser.add_argument('--teamcity-stat-output', action='store_const', const=True) - parser.add_argument('--coverage-report-path', action='store') + parser.add_argument('--coverage-report-path', action='store') parser.add_argument('--gcov-report', action='store') parser.add_argument('--lcov-cobertura', action='store') diff --git a/build/scripts/fetch_from_external.py b/build/scripts/fetch_from_external.py index d4ed6f4221..47060130f8 100644 --- a/build/scripts/fetch_from_external.py +++ b/build/scripts/fetch_from_external.py @@ -1,11 +1,11 @@ -import sys -import json +import sys +import json import os.path import fetch_from import argparse import logging - - + + def parse_args(): parser = argparse.ArgumentParser() fetch_from.add_common_arguments(parser) @@ -13,8 +13,8 @@ def parse_args(): parser.add_argument('--custom-fetcher') parser.add_argument('--resource-file') return parser.parse_args() - - + + def main(args): external_file = args.external_file.rstrip('.external') if os.path.isfile(args.resource_file): @@ -25,7 +25,7 @@ def main(args): try: with open(args.external_file) as f: js = json.load(f) - + if js['storage'] == 'SANDBOX': import fetch_from_sandbox as ffsb del args.external_file @@ -45,7 +45,7 @@ def main(args): raise Exception(error) -if __name__ == '__main__': +if __name__ == '__main__': args = parse_args() fetch_from.setup_logging(args, os.path.basename(__file__)) diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py index 23487f5c1e..709a9b4fbd 100644 --- a/build/scripts/link_dyn_lib.py +++ b/build/scripts/link_dyn_lib.py @@ -30,75 +30,75 @@ def parse_export_file(p): yield {'lang': words[0], 'sym': words[1]} elif len(words) == 1: yield {'lang': 'C', 'sym': words[0]} - else: + else: raise Exception('unsupported exports line: ' + l) -def to_c(sym): - symbols = collections.deque(sym.split('::')) - c_prefixes = [ # demangle prefixes for c++ symbols - '_ZN', # namespace - '_ZTIN', # typeinfo for - '_ZTSN', # typeinfo name for - '_ZTTN', # VTT for - '_ZTVN', # vtable for - '_ZNK', # const methods - ] - c_sym = '' - while symbols: - s = symbols.popleft() - if s == '*': - c_sym += '*' - break - if '*' in s and len(s) > 1: - raise Exception('Unsupported format, cannot guess length of symbol: ' + s) - c_sym += str(len(s)) + s - if symbols: - raise Exception('Unsupported format: ' + sym) - if c_sym[-1] != '*': - c_sym += 'E*' - return ['{prefix}{sym}'.format(prefix=prefix, sym=c_sym) for prefix in c_prefixes] - - +def to_c(sym): + symbols = collections.deque(sym.split('::')) + c_prefixes = [ # demangle prefixes for c++ symbols + '_ZN', # namespace + '_ZTIN', # typeinfo for + '_ZTSN', # typeinfo name for + '_ZTTN', # VTT for + '_ZTVN', # vtable for + '_ZNK', # const methods + ] + c_sym = '' + while symbols: + s = symbols.popleft() + if s == '*': + c_sym += '*' + break + if '*' in s and len(s) > 1: + raise Exception('Unsupported format, cannot guess length of symbol: ' + s) + c_sym += str(len(s)) + s + if symbols: + raise Exception('Unsupported format: ' + sym) + if c_sym[-1] != '*': + c_sym += 'E*' + return ['{prefix}{sym}'.format(prefix=prefix, sym=c_sym) for prefix in c_prefixes] + + def fix_darwin_param(ex): for item in ex: - if item.get('linux_version'): - continue - + if item.get('linux_version'): + continue + if item['lang'] == 'C': yield '-Wl,-exported_symbol,_' + item['sym'] - elif item['lang'] == 'C++': - for sym in to_c(item['sym']): - yield '-Wl,-exported_symbol,_' + sym + elif item['lang'] == 'C++': + for sym in to_c(item['sym']): + yield '-Wl,-exported_symbol,_' + sym else: raise Exception('unsupported lang: ' + item['lang']) def fix_gnu_param(arch, ex): d = collections.defaultdict(list) - version = None + version = None for item in ex: - if item.get('linux_version'): - if not version: - version = item.get('linux_version') - else: - raise Exception('More than one linux_version defined') - elif item['lang'] == 'C++': - d['C'].extend(to_c(item['sym'])) - else: - d[item['lang']].append(item['sym']) + if item.get('linux_version'): + if not version: + version = item.get('linux_version') + else: + raise Exception('More than one linux_version defined') + elif item['lang'] == 'C++': + d['C'].extend(to_c(item['sym'])) + else: + d[item['lang']].append(item['sym']) with tempfile.NamedTemporaryFile(mode='wt', delete=False) as f: - if version: - f.write('{} {{\nglobal:\n'.format(version)) - else: - f.write('{\nglobal:\n') + if version: + f.write('{} {{\nglobal:\n'.format(version)) + else: + f.write('{\nglobal:\n') for k, v in d.items(): f.write(' extern "' + k + '" {\n') for x in v: - f.write(' ' + x + ';\n') + f.write(' ' + x + ';\n') f.write(' };\n') @@ -208,87 +208,87 @@ if __name__ == '__main__': if os.path.exists(opts.soname): os.unlink(opts.soname) os.link(opts.target, opts.soname) - - -# -----------------Test---------------- # -def write_temp_file(content): - import yatest.common as yc - filename = yc.output_path('test.exports') - with open(filename, 'w') as f: - f.write(content) - return filename - - -def test_fix_cmd_darwin(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -C++ geobase5::hardcoded_service -""" - filename = write_temp_file(export_file_content) - args = ['-Wl,--version-script={}'.format(filename)] + + +# -----------------Test---------------- # +def write_temp_file(content): + import yatest.common as yc + filename = yc.output_path('test.exports') + with open(filename, 'w') as f: + f.write(content) + return filename + + +def test_fix_cmd_darwin(): + export_file_content = """ +C++ geobase5::details::lookup_impl::* +C++ geobase5::hardcoded_service +""" + filename = write_temp_file(export_file_content) + args = ['-Wl,--version-script={}'.format(filename)] assert fix_cmd('DARWIN', False, args) == [ - '-Wl,-exported_symbol,__ZN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTIN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTSN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTTN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZTVN8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZNK8geobase57details11lookup_impl*', - '-Wl,-exported_symbol,__ZN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTIN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTSN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTTN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZTVN8geobase517hardcoded_serviceE*', - '-Wl,-exported_symbol,__ZNK8geobase517hardcoded_serviceE*', - ] - - -def run_fix_gnu_param(export_file_content): - filename = write_temp_file(export_file_content) - result = fix_gnu_param('LINUX', list(parse_export_file(filename)))[0] - version_script_path = result[len('-Wl,--version-script='):] - with open(version_script_path) as f: - content = f.read() - return content - - -def test_fix_gnu_param(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -C getFactoryMap -""" - assert run_fix_gnu_param(export_file_content) == """{ -global: - extern "C" { - _ZN8geobase57details11lookup_impl*; - _ZTIN8geobase57details11lookup_impl*; - _ZTSN8geobase57details11lookup_impl*; - _ZTTN8geobase57details11lookup_impl*; - _ZTVN8geobase57details11lookup_impl*; - _ZNK8geobase57details11lookup_impl*; - getFactoryMap; - }; -local: *; -}; -""" - - -def test_fix_gnu_param_with_linux_version(): - export_file_content = """ -C++ geobase5::details::lookup_impl::* -linux_version ver1.0 -C getFactoryMap -""" - assert run_fix_gnu_param(export_file_content) == """ver1.0 { -global: - extern "C" { - _ZN8geobase57details11lookup_impl*; - _ZTIN8geobase57details11lookup_impl*; - _ZTSN8geobase57details11lookup_impl*; - _ZTTN8geobase57details11lookup_impl*; - _ZTVN8geobase57details11lookup_impl*; - _ZNK8geobase57details11lookup_impl*; - getFactoryMap; - }; -local: *; -}; -""" + '-Wl,-exported_symbol,__ZN8geobase57details11lookup_impl*', + '-Wl,-exported_symbol,__ZTIN8geobase57details11lookup_impl*', + '-Wl,-exported_symbol,__ZTSN8geobase57details11lookup_impl*', + '-Wl,-exported_symbol,__ZTTN8geobase57details11lookup_impl*', + '-Wl,-exported_symbol,__ZTVN8geobase57details11lookup_impl*', + '-Wl,-exported_symbol,__ZNK8geobase57details11lookup_impl*', + '-Wl,-exported_symbol,__ZN8geobase517hardcoded_serviceE*', + '-Wl,-exported_symbol,__ZTIN8geobase517hardcoded_serviceE*', + '-Wl,-exported_symbol,__ZTSN8geobase517hardcoded_serviceE*', + '-Wl,-exported_symbol,__ZTTN8geobase517hardcoded_serviceE*', + '-Wl,-exported_symbol,__ZTVN8geobase517hardcoded_serviceE*', + '-Wl,-exported_symbol,__ZNK8geobase517hardcoded_serviceE*', + ] + + +def run_fix_gnu_param(export_file_content): + filename = write_temp_file(export_file_content) + result = fix_gnu_param('LINUX', list(parse_export_file(filename)))[0] + version_script_path = result[len('-Wl,--version-script='):] + with open(version_script_path) as f: + content = f.read() + return content + + +def test_fix_gnu_param(): + export_file_content = """ +C++ geobase5::details::lookup_impl::* +C getFactoryMap +""" + assert run_fix_gnu_param(export_file_content) == """{ +global: + extern "C" { + _ZN8geobase57details11lookup_impl*; + _ZTIN8geobase57details11lookup_impl*; + _ZTSN8geobase57details11lookup_impl*; + _ZTTN8geobase57details11lookup_impl*; + _ZTVN8geobase57details11lookup_impl*; + _ZNK8geobase57details11lookup_impl*; + getFactoryMap; + }; +local: *; +}; +""" + + +def test_fix_gnu_param_with_linux_version(): + export_file_content = """ +C++ geobase5::details::lookup_impl::* +linux_version ver1.0 +C getFactoryMap +""" + assert run_fix_gnu_param(export_file_content) == """ver1.0 { +global: + extern "C" { + _ZN8geobase57details11lookup_impl*; + _ZTIN8geobase57details11lookup_impl*; + _ZTSN8geobase57details11lookup_impl*; + _ZTTN8geobase57details11lookup_impl*; + _ZTVN8geobase57details11lookup_impl*; + _ZNK8geobase57details11lookup_impl*; + getFactoryMap; + }; +local: *; +}; +""" diff --git a/build/scripts/run_msvc_wine.py b/build/scripts/run_msvc_wine.py index 439d1f8831..34862af650 100644 --- a/build/scripts/run_msvc_wine.py +++ b/build/scripts/run_msvc_wine.py @@ -94,7 +94,7 @@ def call_wine_cmd_once(wine, cmd, env, mode): 'Could not load wine-gecko', 'wine: configuration in', 'wine: created the configuration directory', - 'libpng warning:' + 'libpng warning:' ] suffixes = [ @@ -106,13 +106,13 @@ def call_wine_cmd_once(wine, cmd, env, mode): ] substrs = [ - 'Creating library Z:', - 'err:heap', - 'err:menubuilder:', - 'err:msvcrt', - 'err:ole:', - 'err:wincodecs:', - 'err:winediag:', + 'Creating library Z:', + 'err:heap', + 'err:menubuilder:', + 'err:msvcrt', + 'err:ole:', + 'err:wincodecs:', + 'err:winediag:', ] def good_line(l): @@ -318,47 +318,47 @@ def colorize(out): return '\n'.join(colorize_line(l) for l in out.split('\n')) -def trim_path(path, winepath): +def trim_path(path, winepath): p1 = run_subprocess([winepath, '-w', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p1_stdout, p1_stderr = p1.communicate() win_path = p1_stdout.strip() - + if p1.returncode != 0 or not win_path: # Fall back to only winepath -s win_path = path - + p2 = run_subprocess([winepath, '-s', win_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p2_stdout, p2_stderr = p2.communicate() short_path = p2_stdout.strip() - + check_path = short_path if check_path.startswith(('Z:', 'z:')): check_path = check_path[2:] - + if not check_path[1:].startswith((path[1:4], path[1:4].upper())): raise Exception('Cannot trim path {}; 1st winepath exit code: {}, stdout:\n{}\n stderr:\n{}\n 2nd winepath exit code: {}, stdout:\n{}\n stderr:\n{}'.format(path, p1.returncode, p1_stdout, p1_stderr, p2.returncode, p2_stdout, p2_stderr)) return short_path -def downsize_path(path, short_names): - flag = '' - if path.startswith('/Fo'): - flag = '/Fo' - path = path[3:] - - for full_name, short_name in short_names.items(): - if path.startswith(full_name): - path = path.replace(full_name, short_name) - - return flag + path - - -def make_full_path_arg(arg, bld_root, short_root): - if arg[0] != '/' and len(os.path.join(bld_root, arg)) > 250: - return os.path.join(short_root, arg) - return arg - +def downsize_path(path, short_names): + flag = '' + if path.startswith('/Fo'): + flag = '/Fo' + path = path[3:] + + for full_name, short_name in short_names.items(): + if path.startswith(full_name): + path = path.replace(full_name, short_name) + + return flag + path + + +def make_full_path_arg(arg, bld_root, short_root): + if arg[0] != '/' and len(os.path.join(bld_root, arg)) > 250: + return os.path.join(short_root, arg) + return arg + def fix_path(p): topdirs = ['/%s/' % d for d in os.listdir('/')] def abs_path_start(path, pos): @@ -414,8 +414,8 @@ def run_main(): parser.add_argument('-v', action='store', dest='version', default='120') parser.add_argument('-I', action='append', dest='incl_paths') parser.add_argument('mode', action='store') - parser.add_argument('arcadia_root', action='store') - parser.add_argument('arcadia_build_root', action='store') + parser.add_argument('arcadia_root', action='store') + parser.add_argument('arcadia_build_root', action='store') parser.add_argument('binary', action='store') parser.add_argument('free_args', nargs=argparse.REMAINDER) # By now just unpack. Ideally we should fix path and pack arguments back into command file @@ -426,10 +426,10 @@ def run_main(): binary = args.binary version = args.version incl_paths = args.incl_paths - bld_root = args.arcadia_build_root + bld_root = args.arcadia_build_root free_args = args.free_args - wine_dir = os.path.dirname(os.path.dirname(wine)) + wine_dir = os.path.dirname(os.path.dirname(wine)) bin_dir = os.path.dirname(binary) tc_dir = os.path.dirname(os.path.dirname(os.path.dirname(bin_dir))) if not incl_paths: @@ -449,10 +449,10 @@ def run_main(): env['WindowsSdkDir'] = fix_path(tc_dir) env['LIBPATH'] = fix_path(tc_dir + '/VC/lib/amd64') env['LIB'] = fix_path(tc_dir + '/VC/lib/amd64') - env['LD_LIBRARY_PATH'] = ':'.join(wine_dir + d for d in ['/lib', '/lib64', '/lib64/wine']) + env['LD_LIBRARY_PATH'] = ':'.join(wine_dir + d for d in ['/lib', '/lib64', '/lib64/wine']) cmd = [binary] + process_free_args(free_args, wine, bld_root, mode) - + for x in ('/NOLOGO', '/nologo', '/FD'): try: cmd.remove(x) diff --git a/build/scripts/ya.make b/build/scripts/ya.make index 710165e40d..35b7184460 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -1,8 +1,8 @@ OWNER(g:ymake) PY2TEST() - -TEST_SRCS( + +TEST_SRCS( build_dll_and_java.py build_info_gen.py build_java_codenav_index.py @@ -53,7 +53,7 @@ TEST_SRCS( ios_wrapper.py java_pack_to_file.py link_asrc.py - link_dyn_lib.py + link_dyn_lib.py link_exe.py link_fat_obj.py link_lib.py @@ -93,8 +93,8 @@ TEST_SRCS( xargs.py yield_line.py yndexer.py -) - +) + PEERDIR( ydb/library/yql/public/udf ) diff --git a/build/sysincl/misc.yml b/build/sysincl/misc.yml index e9e6095888..2401bf642c 100644 --- a/build/sysincl/misc.yml +++ b/build/sysincl/misc.yml @@ -111,9 +111,9 @@ includes: - regex.h: contrib/tools/tre/lib/regex.h -- source_filter: "^contrib/libs/cppkafka" +- source_filter: "^contrib/libs/cppkafka" includes: - - error.h: contrib/libs/cppkafka/include/cppkafka/error.h + - error.h: contrib/libs/cppkafka/include/cppkafka/error.h - source_filter: "^contrib/libs/clang11-rt" includes: diff --git a/build/ya.conf.json b/build/ya.conf.json index 5f7cc875d6..5fcbd3e2ee 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -775,8 +775,8 @@ "target": { "os": "WIN", "arch": "i686" - }, - "default": true + }, + "default": true } ], "params": { @@ -785,7 +785,7 @@ "werror_mode": "compiler_specific", "c_compiler": "$(MSVC)/bin/Hostx64/x86/cl.exe", "cxx_compiler": "$(MSVC)/bin/Hostx64/x86/cl.exe", - "cxx_std": "c++latest", + "cxx_std": "c++latest", "version": "2019" } }, @@ -805,8 +805,8 @@ { "host": { "os": "WIN" - }, - "default": true + }, + "default": true } ], "params": { @@ -815,7 +815,7 @@ "werror_mode": "compiler_specific", "c_compiler": "$(MSVC)/bin/Hostx64/x64/cl.exe", "cxx_compiler": "$(MSVC)/bin/Hostx64/x64/cl.exe", - "cxx_std": "c++latest", + "cxx_std": "c++latest", "version": "2019", "for_ide": "msvs2019" } @@ -840,8 +840,8 @@ "target": { "os": "WIN", "arch": "arm" - }, - "default": true + }, + "default": true } ], "params": { @@ -850,7 +850,7 @@ "werror_mode": "compiler_specific", "c_compiler": "$(MSVC)/bin/Hostx64/arm/cl.exe", "cxx_compiler": "$(MSVC)/bin/Hostx64/arm/cl.exe", - "cxx_std": "c++latest", + "cxx_std": "c++latest", "version": "2019" } }, @@ -874,8 +874,8 @@ "target": { "os": "WIN", "arch": "i686" - }, - "default": true + }, + "default": true } ], "params": { @@ -884,7 +884,7 @@ "werror_mode": "compiler_specific", "c_compiler": "${YMAKE_PYTHON} ${input:\"build/scripts/run_msvc_wine.py\"} $(WINE_TOOL-sbr:1093314933)/bin/wine64 -v140 c $WINE_ENV ${ARCADIA_ROOT} ${ARCADIA_BUILD_ROOT} $(MSVC)/bin/Hostx64/x86/cl.exe", "cxx_compiler": "${YMAKE_PYTHON} ${input:\"build/scripts/run_msvc_wine.py\"} $(WINE_TOOL-sbr:1093314933)/bin/wine64 -v140 cxx $WINE_ENV ${ARCADIA_ROOT} ${ARCADIA_BUILD_ROOT} $(MSVC)/bin/Hostx64/x86/cl.exe", - "cxx_std": "c++latest", + "cxx_std": "c++latest", "version": "2019", "wine": "yes" } @@ -908,8 +908,8 @@ }, "target": { "os": "WIN" - }, - "default": true + }, + "default": true } ], "params": { @@ -918,7 +918,7 @@ "werror_mode": "compiler_specific", "c_compiler": "${YMAKE_PYTHON} ${input:\"build/scripts/run_msvc_wine.py\"} $(WINE_TOOL-sbr:1093314933)/bin/wine64 -v140 c $WINE_ENV ${ARCADIA_ROOT} ${ARCADIA_BUILD_ROOT} $(MSVC)/bin/Hostx64/x64/cl.exe", "cxx_compiler": "${YMAKE_PYTHON} ${input:\"build/scripts/run_msvc_wine.py\"} $(WINE_TOOL-sbr:1093314933)/bin/wine64 -v140 cxx $WINE_ENV ${ARCADIA_ROOT} ${ARCADIA_BUILD_ROOT} $(MSVC)/bin/Hostx64/x64/cl.exe", - "cxx_std": "c++latest", + "cxx_std": "c++latest", "version": "2019", "wine": "yes" } @@ -943,8 +943,8 @@ "target": { "os": "WIN", "arch": "arm" - }, - "default": true + }, + "default": true } ], "params": { @@ -953,7 +953,7 @@ "werror_mode": "compiler_specific", "c_compiler": "${YMAKE_PYTHON} ${input:\"build/scripts/run_msvc_wine.py\"} $(WINE_TOOL-sbr:1093314933)/bin/wine64 -v140 c $WINE_ENV ${ARCADIA_ROOT} ${ARCADIA_BUILD_ROOT} $(MSVC)/bin/Hostx64/arm/cl.exe", "cxx_compiler": "${YMAKE_PYTHON} ${input:\"build/scripts/run_msvc_wine.py\"} $(WINE_TOOL-sbr:1093314933)/bin/wine64 -v140 cxx $WINE_ENV ${ARCADIA_ROOT} ${ARCADIA_BUILD_ROOT} $(MSVC)/bin/Hostx64/arm/cl.exe", - "cxx_std": "c++latest", + "cxx_std": "c++latest", "version": "2019", "wine": "yes" } @@ -5726,7 +5726,7 @@ "bottles": { "msvc2019": { "formula": { - "sandbox_id": 887495315, + "sandbox_id": 887495315, "match": "Microsoft Visual C++" }, "executable": { @@ -6485,7 +6485,7 @@ }, "wine": { "formula": { - "sandbox_id": 495594294, + "sandbox_id": 495594294, "match": "Wine" }, "executable": { @@ -7295,7 +7295,7 @@ }, "ygdiff": { "formula": { - "sandbox_id": 982695480, + "sandbox_id": 982695480, "match": "ygdiff" }, "executable": { diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 081833998b..1e890229fc 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -276,11 +276,11 @@ PERL_DEFAULT_PEERDIR=yes # tag:perl-specific PERLSUFFIX=-csuffix .cpp -PERLOUTPUT=${stdout;output:SRC.cpp} -when ($PERLC == "yes") { - PERLSUFFIX=-csuffix .c - PERLOUTPUT=${stdout;output:SRC.c} -} +PERLOUTPUT=${stdout;output:SRC.cpp} +when ($PERLC == "yes") { + PERLSUFFIX=-csuffix .c + PERLOUTPUT=${stdout;output:SRC.c} +} XSUBPPFLAGS= # tag:tool-specific @@ -298,7 +298,7 @@ FML_TOOL=${tool:"tools/relev_fml_codegen"} FML_UNUSED_TOOL=${tool:"tools/relev_fml_unused"} LUA_TOOL=${tool:"tools/lua"} ENUM_PARSER_TOOL=${tool:"tools/enum_parser/enum_parser"} - + # tag:python-specific tag:cython-specific CYTHON_SCRIPT=${input:"${ARCADIA_ROOT}/contrib/tools/cython/cython.py"} RUN_CYTHON_SCRIPT=$YMAKE_PYTHON $CYTHON_SCRIPT @@ -345,42 +345,42 @@ FATAL_ERROR_MODULE=no FATAL_ERROR_MESSAGE= # tag:python-specific -# PYTHON_DISTR={ "default" | "macos-brew" } determine python system supply +# PYTHON_DISTR={ "default" | "macos-brew" } determine python system supply # see build/platform/python/ya.make -PYTHON_DISTR="default" -when ($USE_SYSTEM_PYTHON) { - USE_ARCADIA_PYTHON=no +PYTHON_DISTR="default" +when ($USE_SYSTEM_PYTHON) { + USE_ARCADIA_PYTHON=no select ($USE_SYSTEM_PYTHON) { "2.7" ? { _SYSTEM_PYTHON27=yes - PY_VERSION=2.7 - PY_FRAMEWORK_VERSION=2.7 + PY_VERSION=2.7 + PY_FRAMEWORK_VERSION=2.7 } - "3.4" ? { + "3.4" ? { _SYSTEM_PYTHON34=yes - PY_VERSION=3.4m - PY_FRAMEWORK_VERSION=3.4 - } + PY_VERSION=3.4m + PY_FRAMEWORK_VERSION=3.4 + } "3.5" ? { _SYSTEM_PYTHON35=yes - PY_VERSION=3.5m - PY_FRAMEWORK_VERSION=3.5 + PY_VERSION=3.5m + PY_FRAMEWORK_VERSION=3.5 } "3.6" ? { _SYSTEM_PYTHON36=yes - PY_VERSION=3.6m - PY_FRAMEWORK_VERSION=3.6 - } - "3.7" ? { - _SYSTEM_PYTHON37=yes - PY_VERSION=3.7m - PY_FRAMEWORK_VERSION=3.7 - } - "3.8" ? { - _SYSTEM_PYTHON38=yes - PY_VERSION=3.8 - PY_FRAMEWORK_VERSION=3.8 + PY_VERSION=3.6m + PY_FRAMEWORK_VERSION=3.6 } + "3.7" ? { + _SYSTEM_PYTHON37=yes + PY_VERSION=3.7m + PY_FRAMEWORK_VERSION=3.7 + } + "3.8" ? { + _SYSTEM_PYTHON38=yes + PY_VERSION=3.8 + PY_FRAMEWORK_VERSION=3.8 + } "3.9" ? { _SYSTEM_PYTHON39=yes PY_VERSION=3.9 @@ -394,9 +394,9 @@ when ($USE_SYSTEM_PYTHON) { default ? { FATAL_ERROR_MESSAGE+=invalid USE_SYSTEM_PYTHON value } - } -} - + } +} + # tag:python-specific YMAKE_PYTHON3_PEER=build/platform/python/ymake_python3 YMAKE_PYTHON3=$YMAKE_PYTHON3_RESOURCE_GLOBAL/python3 @@ -1215,14 +1215,14 @@ module _BASE_UNIT: _BARE_UNIT { } } - when ($HARDENING == "yes") { - when ($CLANG) { + when ($HARDENING == "yes") { + when ($CLANG) { CFLAGS+=-fstack-protector-all -D_hardening_enabled_ LDFLAGS+=-z relro -z now -z noexecstack PIE=yes - } - } - + } + } + when ($USE_LTO == "yes") { when ($GCC) { CFLAGS+=-flto -fno-fat-lto-objects @@ -2667,15 +2667,15 @@ module PY_ANY_MODULE: DLL_UNIT { LDFLAGS+= -Wl,-Bsymbolic } - when ($USE_ARCADIA_PYTHON == "no") { - when ($USE_SYSTEM_PYTHON) { + when ($USE_ARCADIA_PYTHON == "no") { + when ($USE_SYSTEM_PYTHON) { PEERDIR+=build/platform/python - } - otherwise { - when ($MSVC == "yes" || $CYGWIN == "yes") { - LDFLAGS+=$PYTHON_LIBRARIES - } - } + } + otherwise { + when ($MSVC == "yes" || $CYGWIN == "yes") { + LDFLAGS+=$PYTHON_LIBRARIES + } + } } # This by now replicates ymake's bahavior. We'll get rid of SONAME setting in ymake and fix this code alltogether @@ -2691,7 +2691,7 @@ module PY_ANY_MODULE: DLL_UNIT { } when ($MSVC == "yes" || $CYGWIN == "yes") { - MODULE_PREFIX= + MODULE_PREFIX= MODULE_SUFFIX=.pyd } otherwise { @@ -5124,12 +5124,12 @@ macro _PYTHON_ADDINCL() { CFLAGS+=-DARCADIA_PYTHON_UNICODE_SIZE=$ARCADIA_PYTHON_UNICODE_SIZE } otherwise { - when ($USE_SYSTEM_PYTHON) { + when ($USE_SYSTEM_PYTHON) { PEERDIR+=build/platform/python - } - otherwise { - CFLAGS+=$PYTHON_INCLUDE - } + } + otherwise { + CFLAGS+=$PYTHON_INCLUDE + } } } @@ -5776,7 +5776,7 @@ macro _SRC("sfdl", SRC, SRCFLAGS...) { # tag:src-processing macro _SRC("xs", SRC, SRCFLAGS...) { - .CMD=$PERL_LD_LIBRARY_PATH $PERL $PERL_INCLUDE ${suf=/ExtUtils/xsubpp:PERL_PRIVLIB} -typemap ${suf=/ExtUtils/typemap:PERL_PRIVLIB} $PERLSUFFIX $XSUBPPFLAGS ${SRCFLAGS} ${input:SRC} $PERLOUTPUT ${kv;hide:"p XS"} ${kv;hide:"pc yellow"} + .CMD=$PERL_LD_LIBRARY_PATH $PERL $PERL_INCLUDE ${suf=/ExtUtils/xsubpp:PERL_PRIVLIB} -typemap ${suf=/ExtUtils/typemap:PERL_PRIVLIB} $PERLSUFFIX $XSUBPPFLAGS ${SRCFLAGS} ${input:SRC} $PERLOUTPUT ${kv;hide:"p XS"} ${kv;hide:"pc yellow"} } # tag:src-processing @@ -6297,7 +6297,7 @@ macro BUILDWITH_CYTHON_C(Src, Options...) { .CMD=$RUN_CYTHON_SCRIPT $CYTHON_OPTIONS ${Options} ${pre=-I:_CYTHON__INCLUDE} ${input:Src} -o ${output;tobindir:Src.c} $CYTHON_OUTPUT_INCLUDES ${requirements;hide:PY_REQUIREMENTS} ${kv;hide:"p CY"} ${kv;hide:"pc yellow"} ADDINCL(FOR cython contrib/tools/cython/Cython/Includes) } - + # tag:python-processing tag:cython tag:internal ### @usage: _BUILDWITH_CYTHON_C_DEP(Src Dep Options...) # internal ### @@ -6556,8 +6556,8 @@ macro LDFLAGS(Flags...) { ### Note: remember about the incompatibility flags for clang and cl (to set flags specifically for cl.exe use MSVC_FLAGS). macro CFLAGS(Flags...) { SET_APPEND_WITH_GLOBAL(USER_CFLAGS $Flags) -} - +} + # tag:flags ### @usage: MASMFLAGS(compiler flags) ### Add the specified flags to the compilation command of .masm files. @@ -7317,16 +7317,16 @@ macro _SRC_yasm(SRC, PREINCLUDES[], SRCFLAGS...) { macro ASM_PREINCLUDE(PREINCLUDES...) { SET_APPEND(YASM_PREINCLUDES_VALUE $PREINCLUDES) } - + # tag:python-specific ### @usage: RUN_PYTHON(Args...) ### ### Version of RUN() macro to invoke Python scripts ### @see: [RUN()](#macro_RUN) macro RUN_PYTHON(Args...) { - SETUP_RUN_PYTHON() - RUN(${PYTHON_BIN} $Args) -} + SETUP_RUN_PYTHON() + RUN(${PYTHON_BIN} $Args) +} ### @usage: RUN_ANTLR(Args...) ### diff --git a/build/ymake_conf.py b/build/ymake_conf.py index 30219eb85e..fc847a9a84 100755 --- a/build/ymake_conf.py +++ b/build/ymake_conf.py @@ -634,9 +634,9 @@ class Build(object): python.configure_posix() python.print_variables() - cuda = Cuda(self) - cuda.print_() - CuDNN(cuda).print_() + cuda = Cuda(self) + cuda.print_() + CuDNN(cuda).print_() print_swig_config() @@ -970,8 +970,8 @@ class ToolchainOptions(object): logger.debug('c_compiler=%s', self.c_compiler) logger.debug('cxx_compiler=%s', self.cxx_compiler) - self.compiler_platform_projects = self.target.find_in_dict(self.params.get('platform'), []) - + self.compiler_platform_projects = self.target.find_in_dict(self.params.get('platform'), []) + def version_at_least(self, *args): return args <= tuple(self.compiler_version_list) @@ -1743,10 +1743,10 @@ class SwiftCompiler(object): class Linker(object): - BFD = 'bfd' - LLD = 'lld' - GOLD = 'gold' - + BFD = 'bfd' + LLD = 'lld' + GOLD = 'gold' + def __init__(self, tc, build): """ :type tc: ToolchainOptions @@ -2239,14 +2239,14 @@ class MSVCToolchainOptions(ToolchainOptions): # TODO(somov): Определять автоматически self.version в этом случае else: - if self.version_at_least(2019): - self.sdk_version = '10.0.18362.0' - sdk_dir = '$(WINDOWS_KITS-sbr:1939557911)' - if is_positive('MSVC20'): # XXX: temporary flag, remove after DTCC-123 is completed - self.cxx_std = 'c++latest' - else: - self.sdk_version = '10.0.16299.0' - sdk_dir = '$(WINDOWS_KITS-sbr:1379398385)' + if self.version_at_least(2019): + self.sdk_version = '10.0.18362.0' + sdk_dir = '$(WINDOWS_KITS-sbr:1939557911)' + if is_positive('MSVC20'): # XXX: temporary flag, remove after DTCC-123 is completed + self.cxx_std = 'c++latest' + else: + self.sdk_version = '10.0.16299.0' + sdk_dir = '$(WINDOWS_KITS-sbr:1379398385)' self.vc_root = self.name_marker if not self.use_clang else '$MSVC_FOR_CLANG_RESOURCE_GLOBAL' self.kit_includes = os.path.join(sdk_dir, 'Include', self.sdk_version) @@ -2356,7 +2356,7 @@ class MSVCCompiler(MSVC, Compiler): 4201, # nonstandard extension used : nameless struct/union 4351, # elements of array will be default initialized 4355, # 'this' : used in base member initializer list - 4503, # decorated name length exceeded, name was truncated + 4503, # decorated name length exceeded, name was truncated 4510, # default constructor could not be generated 4511, # copy constructor could not be generated 4512, # assignment operator could not be generated @@ -3148,8 +3148,8 @@ class Cuda(object): 'Y_SDK_Root': '$WINDOWS_KITS_RESOURCE_GLOBAL', } - if not self.build.tc.ide_msvs: - self.peerdirs.append('build/platform/msvc') + if not self.build.tc.ide_msvs: + self.peerdirs.append('build/platform/msvc') self.cuda_host_compiler_env.value = format_env(env) self.cuda_host_msvc_version.value = vc_version return '%(Y_VC_Root)s/bin/HostX64/x64/cl.exe' % env @@ -3176,26 +3176,26 @@ class Cuda(object): self.cuda_host_compiler_env.value = format_env({'Y_VC_Root': vc_root}) -class CuDNN(object): - def __init__(self, cuda): - """ - :type cuda: Cuda - """ - self.cuda = cuda - - self.cudnn_version = Setting('CUDNN_VERSION', auto=self.auto_cudnn_version) - - def have_cudnn(self): - return self.cudnn_version.value in ('7.6.5', '8.0.5') - - def auto_cudnn_version(self): - return '7.6.5' - - def print_(self): - if self.cuda.have_cuda.value and self.have_cudnn(): - self.cudnn_version.emit() - - +class CuDNN(object): + def __init__(self, cuda): + """ + :type cuda: Cuda + """ + self.cuda = cuda + + self.cudnn_version = Setting('CUDNN_VERSION', auto=self.auto_cudnn_version) + + def have_cudnn(self): + return self.cudnn_version.value in ('7.6.5', '8.0.5') + + def auto_cudnn_version(self): + return '7.6.5' + + def print_(self): + if self.cuda.have_cuda.value and self.have_cudnn(): + self.cudnn_version.emit() + + def print_swig_config(): def get_swig_tool(): tool = preset('USER_SWIG_TOOL') |