diff options
| author | say <say@yandex-team.ru> | 2022-02-10 16:48:19 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:48:19 +0300 |
| commit | 2096e85a73bb6b3b20ae25a92943992717fe4167 (patch) | |
| tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
| parent | a6a6f6e1e77c7d7d0cdfad61c093e061d6fb5782 (diff) | |
| download | ydb-2096e85a73bb6b3b20ae25a92943992717fe4167.tar.gz | |
Restoring authorship annotation for <say@yandex-team.ru>. Commit 2 of 2.
20 files changed, 1165 insertions, 1165 deletions
diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index 62cd89b51c..f32a2d39a0 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -1,15 +1,15 @@ import os import collections -from hashlib import md5 +from hashlib import md5 import ymake from _common import stripext, rootrel_arc_src, tobuilddir, listid, resolve_to_ymake_path, generate_chunks, pathid -YA_IDE_VENV_VAR = 'YA_IDE_VENV' -PY_NAMESPACE_PREFIX = 'py/namespace' +YA_IDE_VENV_VAR = 'YA_IDE_VENV' +PY_NAMESPACE_PREFIX = 'py/namespace' BUILTIN_PROTO = 'builtin_proto' - + def is_arc_src(src, unit): return ( src.startswith('${ARCADIA_ROOT}/') or @@ -17,13 +17,13 @@ def is_arc_src(src, unit): unit.resolve_arc_path(src).startswith('$S/') ) -def is_extended_source_search_enabled(path, unit): - if not is_arc_src(path, unit): - return False - if unit.get('NO_EXTENDED_SOURCE_SEARCH') == 'yes': - return False - return True - +def is_extended_source_search_enabled(path, unit): + if not is_arc_src(path, unit): + return False + if unit.get('NO_EXTENDED_SOURCE_SEARCH') == 'yes': + return False + return True + def to_build_root(path, unit): if is_arc_src(path, unit): return '${ARCADIA_BUILD_ROOT}/' + rootrel_arc_src(path, unit) @@ -147,10 +147,10 @@ def is_py3(unit): return unit.get("PYTHON3") == "yes" -def on_py_program(unit, *args): - py_program(unit, is_py3(unit)) - - +def on_py_program(unit, *args): + py_program(unit, is_py3(unit)) + + def py_program(unit, py3): """ Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/#modulpyprogramimakrospymain @@ -194,11 +194,11 @@ def onpy_srcs(unit, *args): upath = unit.path()[3:] py3 = is_py3(unit) - py_main_only = unit.get('PROCESS_PY_MAIN_ONLY') + py_main_only = unit.get('PROCESS_PY_MAIN_ONLY') with_py = not unit.get('PYBUILD_NO_PY') with_pyc = not unit.get('PYBUILD_NO_PYC') in_proto_library = unit.get('PY_PROTO') or unit.get('PY3_PROTO') - venv = unit.get(YA_IDE_VENV_VAR) + venv = unit.get(YA_IDE_VENV_VAR) need_gazetteer_peerdir = False trim = 0 @@ -235,7 +235,7 @@ def onpy_srcs(unit, *args): protos = [] evs = [] fbss = [] - py_namespaces = {} + py_namespaces = {} dump_dir = unit.get('PYTHON_BUILD_DUMP_DIR') dump_output = None @@ -304,25 +304,25 @@ def onpy_srcs(unit, *args): if arg.startswith('/'): ymake.report_configure_error('PY_SRCS item starts with "/": {!r}'.format(arg)) continue - mod_name = stripext(arg).replace('/', '.') - if py3 and path.endswith('.py') and is_extended_source_search_enabled(path, unit): - # Dig out real path from the file path. Unit.path is not enough because of SRCDIR and ADDINCL - root_rel_path = rootrel_arc_src(path, unit) - mod_root_path = root_rel_path[:-(len(path) + 1)] - py_namespaces.setdefault(mod_root_path, set()).add(ns if ns else '.') - mod = ns + mod_name + mod_name = stripext(arg).replace('/', '.') + if py3 and path.endswith('.py') and is_extended_source_search_enabled(path, unit): + # Dig out real path from the file path. Unit.path is not enough because of SRCDIR and ADDINCL + root_rel_path = rootrel_arc_src(path, unit) + mod_root_path = root_rel_path[:-(len(path) + 1)] + py_namespaces.setdefault(mod_root_path, set()).add(ns if ns else '.') + mod = ns + mod_name if main_mod: py_main(unit, mod + ":main") elif py3 and unit_needs_main and main_py: py_main(unit, mod) - if py_main_only: - continue - - if py3 and mod == '__main__': - ymake.report_configure_error('TOP_LEVEL __main__.py is not allowed in PY3_PROGRAM') - + if py_main_only: + continue + + if py3 and mod == '__main__': + ymake.report_configure_error('TOP_LEVEL __main__.py is not allowed in PY3_PROGRAM') + pathmod = (path, mod) if dump_output is not None: @@ -441,28 +441,28 @@ def onpy_srcs(unit, *args): res = [] if py3: - mod_list_md5 = md5() + mod_list_md5 = md5() for path, mod in pys: - mod_list_md5.update(mod) - if not (venv and is_extended_source_search_enabled(path, unit)): - dest = 'py/' + mod.replace('.', '/') + '.py' - if with_py: - res += ['DEST', dest, path] - if with_pyc: - root_rel_path = rootrel_arc_src(path, unit) - dst = path + uniq_suffix(path, unit) - unit.on_py3_compile_bytecode([root_rel_path + '-', path, dst]) - res += ['DEST', dest + '.yapyc3', dst + '.yapyc3'] - - if py_namespaces: - # Note: Add md5 to key to prevent key collision if two or more PY_SRCS() used in the same ya.make - ns_res = [] - for path, ns in sorted(py_namespaces.items()): - key = '{}/{}/{}'.format(PY_NAMESPACE_PREFIX, mod_list_md5.hexdigest(), path) - namespaces = ':'.join(sorted(ns)) - ns_res += ['-', '{}="{}"'.format(key, namespaces)] - unit.onresource(ns_res) - + mod_list_md5.update(mod) + if not (venv and is_extended_source_search_enabled(path, unit)): + dest = 'py/' + mod.replace('.', '/') + '.py' + if with_py: + res += ['DEST', dest, path] + if with_pyc: + root_rel_path = rootrel_arc_src(path, unit) + dst = path + uniq_suffix(path, unit) + unit.on_py3_compile_bytecode([root_rel_path + '-', path, dst]) + res += ['DEST', dest + '.yapyc3', dst + '.yapyc3'] + + if py_namespaces: + # Note: Add md5 to key to prevent key collision if two or more PY_SRCS() used in the same ya.make + ns_res = [] + for path, ns in sorted(py_namespaces.items()): + key = '{}/{}/{}'.format(PY_NAMESPACE_PREFIX, mod_list_md5.hexdigest(), path) + namespaces = ':'.join(sorted(ns)) + ns_res += ['-', '{}="{}"'.format(key, namespaces)] + unit.onresource(ns_res) + unit.onresource_files(res) add_python_lint_checks(unit, 3, [path for path, mod in pys] + unit.get(['_PY_EXTRA_LINT_FILES_VALUE']).split()) else: @@ -583,8 +583,8 @@ def onpy_register(unit, *args): def py_main(unit, arg): - if unit.get('IGNORE_PY_MAIN'): - return + if unit.get('IGNORE_PY_MAIN'): + return unit_needs_main = unit.get('MODULE_TYPE') in ('PROGRAM', 'DLL') if unit_needs_main: py_program(unit, is_py3(unit)) diff --git a/build/prebuilt/contrib/tools/ragel6/ya.make.resource b/build/prebuilt/contrib/tools/ragel6/ya.make.resource index 893016f0a5..e37012f682 100644 --- a/build/prebuilt/contrib/tools/ragel6/ya.make.resource +++ b/build/prebuilt/contrib/tools/ragel6/ya.make.resource @@ -1,9 +1,9 @@ IF (OS_DARWIN AND ARCH_X86_64) - SET(SANDBOX_RESOURCE_ID 1959624378) + SET(SANDBOX_RESOURCE_ID 1959624378) ELSEIF (OS_LINUX AND ARCH_X86_64) - SET(SANDBOX_RESOURCE_ID 1969915297) + SET(SANDBOX_RESOURCE_ID 1969915297) ELSEIF (OS_WINDOWS AND ARCH_X86_64) - SET(SANDBOX_RESOURCE_ID 1959624268) + SET(SANDBOX_RESOURCE_ID 1959624268) ELSE() SET(SANDBOX_RESOURCE_ID) ENDIF() diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 501760d62f..081833998b 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -59,7 +59,7 @@ LIB_REQUIREMENTS=$DEFAULT_REQUIREMENTS LD_REQUIREMENTS=$DEFAULT_REQUIREMENTS JAVA_REQUIREMENTS=$DEFAULT_REQUIREMENTS PY_REQUIREMENTS=$DEFAULT_REQUIREMENTS -BISON_DATA_DIR=contrib/tools/bison/bison/data +BISON_DATA_DIR=contrib/tools/bison/bison/data CPP_BISON_SKELS=${BISON_DATA_DIR}/glr.cc ${BISON_DATA_DIR}/lalr1.cc ${BISON_DATA_DIR}/yacc.c ${BISON_DATA_DIR}/stack.hh ${BISON_DATA_DIR}/variant.hh ${BISON_DATA_DIR}/c++.m4 ${BISON_DATA_DIR}/c++-skel.m4 USE_PREBUILT_TOOLS=yes @@ -1710,18 +1710,18 @@ module PY2_PROGRAM: _PY2_PROGRAM { } # tag:python-specific -### @usage: NO_EXTENDED_SOURCE_SEARCH() -### -### Prevent module using in extended python source search. -### Use the macro if module contains python2-only files (or other python sources which shouldn't be imported by python3 interpreter) -### which resides in the same directories with python 3 usefull code. contrib/python/future is a example. -### Anyway, preferred way is to move such files into separate dir and don't use this macro at all. -### -### Also see: https://docs.yandex-team.ru/ya-make/manual/python/vars#y_python_extended_source_search for details -macro NO_EXTENDED_SOURCE_SEARCH() { - SET(NO_EXTENDED_SOURCE_SEARCH yes) -} - +### @usage: NO_EXTENDED_SOURCE_SEARCH() +### +### Prevent module using in extended python source search. +### Use the macro if module contains python2-only files (or other python sources which shouldn't be imported by python3 interpreter) +### which resides in the same directories with python 3 usefull code. contrib/python/future is a example. +### Anyway, preferred way is to move such files into separate dir and don't use this macro at all. +### +### Also see: https://docs.yandex-team.ru/ya-make/manual/python/vars#y_python_extended_source_search for details +macro NO_EXTENDED_SOURCE_SEARCH() { + SET(NO_EXTENDED_SOURCE_SEARCH yes) +} + # tag:python-specific ### @usage: PY3_PROGRAM([progname]) ### @@ -1730,21 +1730,21 @@ macro NO_EXTENDED_SOURCE_SEARCH() { ### This only compatible with PYTHON3-tagged modules and selects those from multimodules. ### ### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/ -multimodule PY3_PROGRAM { - module PY3_BIN: PY3_PROGRAM_BIN { - .PEERDIRSELF=PY3_BIN_LIB - .IGNORED=RESOURCE RESOURCE_FILES PEERDIR TASKLET_REG - # Notify pybuild to skip almost all PY_SRCS arguments except MAIN - ENABLE(PROCESS_PY_MAIN_ONLY) - _PY_PROGRAM() - } - - module PY3_BIN_LIB: PY3_LIBRARY { +multimodule PY3_PROGRAM { + module PY3_BIN: PY3_PROGRAM_BIN { + .PEERDIRSELF=PY3_BIN_LIB + .IGNORED=RESOURCE RESOURCE_FILES PEERDIR TASKLET_REG + # Notify pybuild to skip almost all PY_SRCS arguments except MAIN + ENABLE(PROCESS_PY_MAIN_ONLY) + _PY_PROGRAM() + } + + module PY3_BIN_LIB: PY3_LIBRARY { .IGNORED=RESTRICT_LICENSES - # Notify pybuild to skip all python main function definitions - ENABLE(IGNORE_PY_MAIN) - _REQUIRE_EXPLICIT_LICENSE() - } + # Notify pybuild to skip all python main function definitions + ENABLE(IGNORE_PY_MAIN) + _REQUIRE_EXPLICIT_LICENSE() + } } # tag:ycr-specific @@ -1792,7 +1792,7 @@ PYCR_LINK_EXE=$LINK_EXE && $_PYCR_GENERATE_CONFIGS($MODULE_PREFIX$REALPRJNAME $M ### ### pycare-specific program module. Generates pycare configs in addition to producing the program. ### If name is not specified it will be generated from the name of the containing project directory. -module PYCR_PROGRAM: PY3_PROGRAM_BIN { +module PYCR_PROGRAM: PY3_PROGRAM_BIN { .CMD=PYCR_LINK_EXE } @@ -2205,12 +2205,12 @@ module PY3TEST_BIN: _BASE_PY3_PROGRAM { } # tag:python-specific tag:test -### Disable submodules with the specified name(s). -### Unlike EXCLUDE_TAGS this variable: -### - affects all projects in the build; -### - filter by module name and ignore MODULE_TAG. -EXCLUDE_SUBMODULES=PY3TEST_LIBRARY - +### Disable submodules with the specified name(s). +### Unlike EXCLUDE_TAGS this variable: +### - affects all projects in the build; +### - filter by module name and ignore MODULE_TAG. +EXCLUDE_SUBMODULES=PY3TEST_LIBRARY + # tag:python-specific tag:test ### @usage: PY3TEST([name]) ### @@ -2221,19 +2221,19 @@ EXCLUDE_SUBMODULES=PY3TEST_LIBRARY ### ### Documentation: https://wiki.yandex-team.ru/yatool/test/#testynapytest ### Documentation about the Arcadia test system: https://wiki.yandex-team.ru/yatool/test/ -multimodule PY3TEST { - module PY3TEST_PROGRAM: PY3TEST_BIN { - .FINAL_TARGET=yes - } - +multimodule PY3TEST { + module PY3TEST_PROGRAM: PY3TEST_BIN { + .FINAL_TARGET=yes + } + module PY3TEST_LIBRARY: _PY3_LIBRARY { - PEERDIR+=library/python/pytest - _REQUIRE_EXPLICIT_LICENSE() - } + PEERDIR+=library/python/pytest + _REQUIRE_EXPLICIT_LICENSE() + } } # tag:cpp-specific tag:test -module CPP_STYLE_TEST: PY3TEST_BIN { +module CPP_STYLE_TEST: PY3TEST_BIN { DEPENDS(contrib/libs/clang12/tools/clang-format) PEERDIR+=library/python/cpp_test } @@ -4211,16 +4211,16 @@ module _BASE_PY3_PROGRAM: _BASE_PROGRAM { } # tag:python-specific -### @usage: PY3_PROGRAM_BIN([progname]) -### Use instead of PY3_PROGRAM only if ya.make with PY3_PROGRAM() included in another ya.make -### In all other cases use PY3_PROGRAM -module PY3_PROGRAM_BIN: _BASE_PY3_PROGRAM { +### @usage: PY3_PROGRAM_BIN([progname]) +### Use instead of PY3_PROGRAM only if ya.make with PY3_PROGRAM() included in another ya.make +### In all other cases use PY3_PROGRAM +module PY3_PROGRAM_BIN: _BASE_PY3_PROGRAM { .ALIASES=REQUIREMENTS=PY_REQUIREMENTS # Look's like we cannot avoid copy-paste util ymake supports multiple inheritance # We need to attach coverage.extractor to every py_program target, except pytest targets ADD_YTEST($MODULE_PREFIX$REALPRJNAME coverage.extractor) -} - +} + # tag:java-specific JAVA_SWIG_DELIM=JAVA_SWIG_DELIM @@ -5061,7 +5061,7 @@ multimodule SANDBOX_PY23_TASK { ### ### Documentation: https://wiki.yandex-team.ru/sandbox/tasks/binary multimodule SANDBOX_PY3_TASK { - module SB_TASK_BIN: PY3_PROGRAM_BIN { + module SB_TASK_BIN: PY3_PROGRAM_BIN { PY_MAIN(sandbox.taskbox.binary) PEERDIR(sandbox/bin sandbox/sdk2 sandbox/taskbox/worker) SET_APPEND(NO_CHECK_IMPORTS_FOR_VALUE api.*) @@ -5158,7 +5158,7 @@ macro PYTHON3_ADDINCL() { macro _ARCADIA_PYTHON3_ADDINCL() { _PYTHON3_ADDINCL() SET(MODULE_TAG PY3) - SET(MODULE_LANG PY3) + SET(MODULE_LANG PY3) SET(PEERDIR_TAGS PY3 PY3_BIN_LIB PY3TEST_LIBRARY PY3_NATIVE PY3_PROTO PY3_FBS PY3_SSQLS YQL_UDF_STATIC __EMPTY__ DLL_LIB) } @@ -5782,7 +5782,7 @@ macro _SRC("xs", SRC, SRCFLAGS...) { # tag:src-processing macro _SRC("y", SRC, SRCFLAGS...) { # .CMD=${tool:"contrib/tools/byacc"} $BYACC_FLAGS ${nopath;noext;output:SRC.cpp} ${input:SRC} ${nopath;noext;hide;output;addincl:SRC.h} ${kv;hide:"p YC"} ${kv;hide:"pc light-green"} - .CMD=${tool:"contrib/tools/bison/bison"} $BISON_FLAGS --m4=${tool:"contrib/tools/bison/m4"} -d -o ${nopath;noext;output;main;hide:SRC.h} ${nopath;noext;output:SRC.cpp} ${input:SRC} ${SRCFLAGS} && $YMAKE_PYTHON ${input:"build/scripts/preprocess.py"} $_ADD_HIDDEN_INPUTS($CPP_BISON_SKELS) ${nopath;noext;output;addincl:SRC.h} ${kv;hide:"p YC"} ${kv;hide:"pc light-green"} + .CMD=${tool:"contrib/tools/bison/bison"} $BISON_FLAGS --m4=${tool:"contrib/tools/bison/m4"} -d -o ${nopath;noext;output;main;hide:SRC.h} ${nopath;noext;output:SRC.cpp} ${input:SRC} ${SRCFLAGS} && $YMAKE_PYTHON ${input:"build/scripts/preprocess.py"} $_ADD_HIDDEN_INPUTS($CPP_BISON_SKELS) ${nopath;noext;output;addincl:SRC.h} ${kv;hide:"p YC"} ${kv;hide:"pc light-green"} } # tag:src-processing @@ -5797,7 +5797,7 @@ macro _SRC("gperf", SRC, SRCFLAGS...) { # tag:src-processing macro _SRC("rl", SRC, SRCFLAGS...) { - .CMD=$RUN_NO_SANITIZE ${tool:"contrib/tools/ragel5/ragel"} $RAGEL_FLAGS ${SRCFLAGS} -o ${tmp:SRC.tmp} ${input:SRC} && $RUN_NO_SANITIZE ${tool:"contrib/tools/ragel5/rlgen-cd"} $RLGEN_FLAGS -o ${output;nopath;noext;defext=.rl5.cpp:SRC} ${tmp:SRC.tmp} ${kv;hide:"p R5"} ${kv;hide:"pc yellow"} + .CMD=$RUN_NO_SANITIZE ${tool:"contrib/tools/ragel5/ragel"} $RAGEL_FLAGS ${SRCFLAGS} -o ${tmp:SRC.tmp} ${input:SRC} && $RUN_NO_SANITIZE ${tool:"contrib/tools/ragel5/rlgen-cd"} $RLGEN_FLAGS -o ${output;nopath;noext;defext=.rl5.cpp:SRC} ${tmp:SRC.tmp} ${kv;hide:"p R5"} ${kv;hide:"pc yellow"} } # tag:src-processing @@ -5995,42 +5995,42 @@ macro _SRC_c_nodeps(SRC, OUTFILE, INC...) { .CMD=$_SRC_C_NODEPS_CMD } -# Custom flags for generated cpp-files. To support another generator: -# - insert additional file extension before .cpp into generated cpp-file name -# - update _LANG_CFLAGS_FILTER variable. Don't forget to add ' SKIP ' after flag list +# Custom flags for generated cpp-files. To support another generator: +# - insert additional file extension before .cpp into generated cpp-file name +# - update _LANG_CFLAGS_FILTER variable. Don't forget to add ' SKIP ' after flag list # # ragel5 and ragel6 generated terrible code which makes use of goto's in switch statements. # This triggers -Werror-implicit-fallthrough due to `unannotated fall-through between switch labels`. # # cython generated code also fails to pass this diagnostics due to `fallthrough annotation in unreachable code`. # We use cython==0.29.26 at the time. This issue might be fixed in further versions. - -## tag:src-processing -_LANG_CFLAGS_FILTER=\ - ${pre=-Wno-implicit-fallthrough SKIP ;ext=.rl5:SRC} \ + +## tag:src-processing +_LANG_CFLAGS_FILTER=\ + ${pre=-Wno-implicit-fallthrough SKIP ;ext=.rl5:SRC} \ ${pre=-Wno-implicit-fallthrough SKIP ;ext=.rl6:SRC} \ ${pre=-Wno-implicit-fallthrough SKIP ;ext=.pyx:SRC} - + +# tag:src-processing +# Magic macro for removing file name from result (file name is passed as unused SKIP parameter) +macro _FILTER_EXTS(SKIP="", FLAGS...) { + .CMD=$FLAGS +} + # tag:src-processing -# Magic macro for removing file name from result (file name is passed as unused SKIP parameter) -macro _FILTER_EXTS(SKIP="", FLAGS...) { - .CMD=$FLAGS -} - -# tag:src-processing -macro _LANG_CFLAGS(SRC) { - .CMD=$_FILTER_EXTS($_LANG_CFLAGS_FILTER) -} - -# tag:src-processing -# ymake bug workaround: variables followed a macros call in .CMD are not substituted and are placed in result as is -# Pack macro call into a variable and use it in _SRC_CPP_CMD instead of macro call +macro _LANG_CFLAGS(SRC) { + .CMD=$_FILTER_EXTS($_LANG_CFLAGS_FILTER) +} + +# tag:src-processing +# ymake bug workaround: variables followed a macros call in .CMD are not substituted and are placed in result as is +# Pack macro call into a variable and use it in _SRC_CPP_CMD instead of macro call _LANG_CFLAGS_VALUE= when ($CLANG == "yes" || $GCC == "yes") { _LANG_CFLAGS_VALUE=$_LANG_CFLAGS(${noext:SRC}) } - -# tag:src-processing + +# tag:src-processing macro _SRC_cpp(SRC, COMPILE_OUT_SUFFIX="", SRCFLAGS...) { .CMD=$_SRC_CPP_CMD } @@ -9022,18 +9022,18 @@ multimodule PY23_TEST { CANONIZE_SUB_PATH=py2test RUN_CYTHON_SCRIPT=$YMAKE_PYTHON $CYTHON_SCRIPT } - module PY3TEST_PROGRAM: PY3TEST_BIN { - .FINAL_TARGET=yes + module PY3TEST_PROGRAM: PY3TEST_BIN { + .FINAL_TARGET=yes OBJ_SUF=.py3 CANONIZE_SUB_PATH=py3test RUN_CYTHON_SCRIPT=$YMAKE_PYTHON $CYTHON_SCRIPT } - + module PY3TEST_LIBRARY: _PY3_LIBRARY { - PEERDIR+=library/python/pytest - _REQUIRE_EXPLICIT_LICENSE() + PEERDIR+=library/python/pytest + _REQUIRE_EXPLICIT_LICENSE() RUN_CYTHON_SCRIPT=$YMAKE_PYTHON $CYTHON_SCRIPT - } + } } # tag:windows-specific diff --git a/build/ymake_conf.py b/build/ymake_conf.py index 0499ace985..30219eb85e 100755 --- a/build/ymake_conf.py +++ b/build/ymake_conf.py @@ -1677,7 +1677,7 @@ class GnuCompiler(Compiler): '$COMPILER_TIME_TRACE_FLAGS', '$EXTRA_OUTPUT', '$SRCFLAGS', - '$_LANG_CFLAGS_VALUE', + '$_LANG_CFLAGS_VALUE', '${input:SRC}', '$TOOLCHAIN_ENV', '$YNDEXER_OUTPUT', diff --git a/contrib/python/future/ya.make b/contrib/python/future/ya.make index 5a00554eb4..ba24f13341 100644 --- a/contrib/python/future/ya.make +++ b/contrib/python/future/ya.make @@ -12,7 +12,7 @@ NO_CHECK_IMPORTS( ) NO_LINT() -NO_EXTENDED_SOURCE_SEARCH() +NO_EXTENDED_SOURCE_SEARCH() PY_SRCS( TOP_LEVEL diff --git a/contrib/python/mypy-protobuf/bin/protoc-gen-mypy/bin/ya.make b/contrib/python/mypy-protobuf/bin/protoc-gen-mypy/bin/ya.make index 36df1a1212..b63a9d4a51 100644 --- a/contrib/python/mypy-protobuf/bin/protoc-gen-mypy/bin/ya.make +++ b/contrib/python/mypy-protobuf/bin/protoc-gen-mypy/bin/ya.make @@ -1,6 +1,6 @@ OWNER(torkve g:python-contrib) -PY3_PROGRAM_BIN(protoc-gen-mypy) +PY3_PROGRAM_BIN(protoc-gen-mypy) PEERDIR( contrib/python/mypy-protobuf diff --git a/contrib/tools/python3/pycc/bin/ya.make b/contrib/tools/python3/pycc/bin/ya.make index 697510b203..c63399c525 100644 --- a/contrib/tools/python3/pycc/bin/ya.make +++ b/contrib/tools/python3/pycc/bin/ya.make @@ -1,6 +1,6 @@ OWNER(g:contrib g:ymake orivej) -PY3_PROGRAM_BIN(pycc) +PY3_PROGRAM_BIN(pycc) ENABLE(PYBUILD_NO_PYC) diff --git a/contrib/tools/python3/src/Include/pyconfig.h b/contrib/tools/python3/src/Include/pyconfig.h index 7924e7fb45..965b9e1be4 100644 --- a/contrib/tools/python3/src/Include/pyconfig.h +++ b/contrib/tools/python3/src/Include/pyconfig.h @@ -1,7 +1,7 @@ #pragma once #ifdef Py_BUILD_CORE -#define ABIFLAGS "" +#define ABIFLAGS "" #define PREFIX "/var/empty" #define EXEC_PREFIX "/var/empty" #define VERSION "3.9" @@ -11,11 +11,11 @@ #if defined(__linux__) #define PLATFORM "linux" #define MULTIARCH "x86_64-linux-gnu" -#define SOABI "cpython-39-x86_64-linux-gnu" +#define SOABI "cpython-39-x86_64-linux-gnu" #elif defined(__APPLE__) #define PLATFORM "darwin" #define MULTIARCH "darwin" -#define SOABI "cpython-39-darwin" +#define SOABI "cpython-39-darwin" #endif #endif diff --git a/contrib/tools/python3/src/Lib/venv/scripts/common/Activate.ps1 b/contrib/tools/python3/src/Lib/venv/scripts/common/Activate.ps1 index bc4dcbbe16..7bc3e7c698 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/common/Activate.ps1 +++ b/contrib/tools/python3/src/Lib/venv/scripts/common/Activate.ps1 @@ -1,241 +1,241 @@ -<# -.Synopsis -Activate a Python virtual environment for the current PowerShell session. - -.Description -Pushes the python executable for a virtual environment to the front of the -$Env:PATH environment variable and sets the prompt to signify that you are -in a Python virtual environment. Makes use of the command line switches as -well as the `pyvenv.cfg` file values present in the virtual environment. - -.Parameter VenvDir -Path to the directory that contains the virtual environment to activate. The -default value for this is the parent of the directory that the Activate.ps1 -script is located within. - -.Parameter Prompt -The prompt prefix to display when this virtual environment is activated. By -default, this prompt is the name of the virtual environment folder (VenvDir) -surrounded by parentheses and followed by a single space (ie. '(.venv) '). - -.Example -Activate.ps1 -Activates the Python virtual environment that contains the Activate.ps1 script. - -.Example -Activate.ps1 -Verbose -Activates the Python virtual environment that contains the Activate.ps1 script, -and shows extra information about the activation as it executes. - -.Example -Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv -Activates the Python virtual environment located in the specified location. - -.Example -Activate.ps1 -Prompt "MyPython" -Activates the Python virtual environment that contains the Activate.ps1 script, -and prefixes the current prompt with the specified string (surrounded in -parentheses) while the virtual environment is active. - -.Notes -On Windows, it may be required to enable this Activate.ps1 script by setting the -execution policy for the user. You can do this by issuing the following PowerShell -command: - -PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - -For more information on Execution Policies: -https://go.microsoft.com/fwlink/?LinkID=135170 - -#> -Param( - [Parameter(Mandatory = $false)] - [String] - $VenvDir, - [Parameter(Mandatory = $false)] - [String] - $Prompt -) - -<# Function declarations --------------------------------------------------- #> - -<# -.Synopsis -Remove all shell session elements added by the Activate script, including the -addition of the virtual environment's Python executable from the beginning of -the PATH variable. - -.Parameter NonDestructive -If present, do not remove this function from the global namespace for the -session. - -#> -function global:deactivate ([switch]$NonDestructive) { - # Revert to original values - - # The prior prompt: - if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { - Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt - Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT - } - - # The prior PYTHONHOME: - if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { - Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME - Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME - } - - # The prior PATH: - if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { - Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH - Remove-Item -Path Env:_OLD_VIRTUAL_PATH - } - - # Just remove the VIRTUAL_ENV altogether: - if (Test-Path -Path Env:VIRTUAL_ENV) { - Remove-Item -Path env:VIRTUAL_ENV - } - - # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: - if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { - Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force - } - - # Leave deactivate function in the global namespace if requested: - if (-not $NonDestructive) { - Remove-Item -Path function:deactivate - } -} - -<# -.Description -Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the -given folder, and returns them in a map. - -For each line in the pyvenv.cfg file, if that line can be parsed into exactly -two strings separated by `=` (with any amount of whitespace surrounding the =) -then it is considered a `key = value` line. The left hand string is the key, -the right hand is the value. - -If the value starts with a `'` or a `"` then the first and last character is -stripped from the value before being captured. - -.Parameter ConfigDir -Path to the directory that contains the `pyvenv.cfg` file. -#> -function Get-PyVenvConfig( - [String] - $ConfigDir -) { - Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" - - # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). - $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue - - # An empty map will be returned if no config file is found. - $pyvenvConfig = @{ } - - if ($pyvenvConfigPath) { - - Write-Verbose "File exists, parse `key = value` lines" - $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath - - $pyvenvConfigContent | ForEach-Object { - $keyval = $PSItem -split "\s*=\s*", 2 - if ($keyval[0] -and $keyval[1]) { - $val = $keyval[1] - - # Remove extraneous quotations around a string value. - if ("'""".Contains($val.Substring(0, 1))) { - $val = $val.Substring(1, $val.Length - 2) - } - - $pyvenvConfig[$keyval[0]] = $val - Write-Verbose "Adding Key: '$($keyval[0])'='$val'" - } - } - } - return $pyvenvConfig -} - - -<# Begin Activate script --------------------------------------------------- #> - -# Determine the containing directory of this script -$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$VenvExecDir = Get-Item -Path $VenvExecPath - -Write-Verbose "Activation script is located in path: '$VenvExecPath'" -Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" -Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" - -# Set values required in priority: CmdLine, ConfigFile, Default -# First, get the location of the virtual environment, it might not be -# VenvExecDir if specified on the command line. -if ($VenvDir) { - Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" -} -else { - Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." - $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") - Write-Verbose "VenvDir=$VenvDir" -} - -# Next, read the `pyvenv.cfg` file to determine any required value such -# as `prompt`. -$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir - -# Next, set the prompt from the command line, or the config file, or -# just use the name of the virtual environment folder. -if ($Prompt) { - Write-Verbose "Prompt specified as argument, using '$Prompt'" -} -else { - Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" - if ($pyvenvCfg -and $pyvenvCfg['prompt']) { - Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" - $Prompt = $pyvenvCfg['prompt']; - } - else { - Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" - Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" - $Prompt = Split-Path -Path $venvDir -Leaf - } -} - -Write-Verbose "Prompt = '$Prompt'" -Write-Verbose "VenvDir='$VenvDir'" - -# Deactivate any currently active virtual environment, but leave the -# deactivate function in place. -deactivate -nondestructive - -# Now set the environment variable VIRTUAL_ENV, used by many tools to determine -# that there is an activated venv. -$env:VIRTUAL_ENV = $VenvDir - -if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { - - Write-Verbose "Setting prompt to '$Prompt'" - - # Set the prompt to include the env name - # Make sure _OLD_VIRTUAL_PROMPT is global - function global:_OLD_VIRTUAL_PROMPT { "" } - Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT - New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt - - function global:prompt { - Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " - _OLD_VIRTUAL_PROMPT - } -} - -# Clear PYTHONHOME -if (Test-Path -Path Env:PYTHONHOME) { - Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME - Remove-Item -Path Env:PYTHONHOME -} - -# Add the venv to the PATH -Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH -$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/contrib/tools/python3/src/Lib/venv/scripts/common/activate b/contrib/tools/python3/src/Lib/venv/scripts/common/activate index 886b27534c..45af3536aa 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/common/activate +++ b/contrib/tools/python3/src/Lib/venv/scripts/common/activate @@ -1,66 +1,66 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - # reset old environment variables - if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then - PATH="${_OLD_VIRTUAL_PATH:-}" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then - PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi - - if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then - PS1="${_OLD_VIRTUAL_PS1:-}" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - if [ ! "${1:-}" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelevant variables -deactivate nondestructive - -VIRTUAL_ENV="__VENV_DIR__" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" -export PATH - -# unset PYTHONHOME if set -# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) -# could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "${PYTHONHOME:-}" ] ; then - _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" - unset PYTHONHOME -fi - -if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then - _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="__VENV_PROMPT__${PS1:-}" - export PS1 -fi - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="__VENV_DIR__" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="__VENV_PROMPT__${PS1:-}" + export PS1 +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null +fi diff --git a/contrib/tools/python3/src/Lib/venv/scripts/nt/activate.bat b/contrib/tools/python3/src/Lib/venv/scripts/nt/activate.bat index 4c2b3389fa..af4c7e0aba 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/nt/activate.bat +++ b/contrib/tools/python3/src/Lib/venv/scripts/nt/activate.bat @@ -1,33 +1,33 @@ -@echo off - -rem This file is UTF-8 encoded, so we need to update the current code page while executing it -for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do ( - set _OLD_CODEPAGE=%%a -) -if defined _OLD_CODEPAGE ( - "%SystemRoot%\System32\chcp.com" 65001 > nul -) - -set VIRTUAL_ENV=__VENV_DIR__ - -if not defined PROMPT set PROMPT=$P$G - -if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% -if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% - -set _OLD_VIRTUAL_PROMPT=%PROMPT% -set PROMPT=__VENV_PROMPT__%PROMPT% - -if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% -set PYTHONHOME= - -if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% -if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% - -set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% - -:END -if defined _OLD_CODEPAGE ( - "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul - set _OLD_CODEPAGE= -) +@echo off + +rem This file is UTF-8 encoded, so we need to update the current code page while executing it +for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do ( + set _OLD_CODEPAGE=%%a +) +if defined _OLD_CODEPAGE ( + "%SystemRoot%\System32\chcp.com" 65001 > nul +) + +set VIRTUAL_ENV=__VENV_DIR__ + +if not defined PROMPT set PROMPT=$P$G + +if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% +if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% + +set _OLD_VIRTUAL_PROMPT=%PROMPT% +set PROMPT=__VENV_PROMPT__%PROMPT% + +if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% +set PYTHONHOME= + +if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% +if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% + +set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% + +:END +if defined _OLD_CODEPAGE ( + "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul + set _OLD_CODEPAGE= +) diff --git a/contrib/tools/python3/src/Lib/venv/scripts/nt/deactivate.bat b/contrib/tools/python3/src/Lib/venv/scripts/nt/deactivate.bat index 954a67cce8..1205c61868 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/nt/deactivate.bat +++ b/contrib/tools/python3/src/Lib/venv/scripts/nt/deactivate.bat @@ -1,21 +1,21 @@ -@echo off - -if defined _OLD_VIRTUAL_PROMPT ( - set "PROMPT=%_OLD_VIRTUAL_PROMPT%" -) -set _OLD_VIRTUAL_PROMPT= - -if defined _OLD_VIRTUAL_PYTHONHOME ( - set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" - set _OLD_VIRTUAL_PYTHONHOME= -) - -if defined _OLD_VIRTUAL_PATH ( - set "PATH=%_OLD_VIRTUAL_PATH%" -) - -set _OLD_VIRTUAL_PATH= - -set VIRTUAL_ENV= - -:END +@echo off + +if defined _OLD_VIRTUAL_PROMPT ( + set "PROMPT=%_OLD_VIRTUAL_PROMPT%" +) +set _OLD_VIRTUAL_PROMPT= + +if defined _OLD_VIRTUAL_PYTHONHOME ( + set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" + set _OLD_VIRTUAL_PYTHONHOME= +) + +if defined _OLD_VIRTUAL_PATH ( + set "PATH=%_OLD_VIRTUAL_PATH%" +) + +set _OLD_VIRTUAL_PATH= + +set VIRTUAL_ENV= + +:END diff --git a/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.csh b/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.csh index d70259e92b..68a0dc74e1 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.csh +++ b/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.csh @@ -1,25 +1,25 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi <davidedb@gmail.com>. -# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com> - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' - -# Unset irrelevant variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "__VENV_DIR__" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" - - -set _OLD_VIRTUAL_PROMPT="$prompt" - -if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "__VENV_PROMPT__$prompt" -endif - -alias pydoc python -m pydoc - -rehash +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi <davidedb@gmail.com>. +# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com> + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "__VENV_DIR__" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "__VENV_PROMPT__$prompt" +endif + +alias pydoc python -m pydoc + +rehash diff --git a/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.fish b/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.fish index ac879903f6..54b9ea5676 100644 --- a/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.fish +++ b/contrib/tools/python3/src/Lib/venv/scripts/posix/activate.fish @@ -1,64 +1,64 @@ -# This file must be used with "source <venv>/bin/activate.fish" *from fish* -# (https://fishshell.com/); you cannot run it directly. - -function deactivate -d "Exit virtual environment and return to normal shell environment" - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt - set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt - end - - set -e VIRTUAL_ENV - if test "$argv[1]" != "nondestructive" - # Self-destruct! - functions -e deactivate - end -end - -# Unset irrelevant variables. -deactivate nondestructive - -set -gx VIRTUAL_ENV "__VENV_DIR__" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH - -# Unset PYTHONHOME if set. -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # fish uses a function instead of an env var to generate the prompt. - - # Save the current fish_prompt function as the function _old_fish_prompt. - functions -c fish_prompt _old_fish_prompt - - # With the original prompt function renamed, we can override with our own. - function fish_prompt - # Save the return status of the last command. - set -l old_status $status - - # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) - - # Restore the return status of the previous command. - echo "exit $old_status" | . - # Output the original/"old" prompt. - _old_fish_prompt - end - - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" -end +# This file must be used with "source <venv>/bin/activate.fish" *from fish* +# (https://fishshell.com/); you cannot run it directly. + +function deactivate -d "Exit virtual environment and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + functions -e fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + + set -e VIRTUAL_ENV + if test "$argv[1]" != "nondestructive" + # Self-destruct! + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "__VENV_DIR__" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH + +# Unset PYTHONHOME if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # Save the current fish_prompt function as the function _old_fish_prompt. + functions -c fish_prompt _old_fish_prompt + + # With the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command. + set -l old_status $status + + # Output the venv prompt; color taken from the blue of the Python logo. + printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + + # Restore the return status of the previous command. + echo "exit $old_status" | . + # Output the original/"old" prompt. + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" +end diff --git a/library/python/runtime_py3/importer.pxi b/library/python/runtime_py3/importer.pxi index 8d8b523d4c..904f94dea2 100644 --- a/library/python/runtime_py3/importer.pxi +++ b/library/python/runtime_py3/importer.pxi @@ -2,7 +2,7 @@ import marshal import sys from _codecs import utf_8_decode, utf_8_encode from _frozen_importlib import _call_with_frames_removed, spec_from_loader, BuiltinImporter -from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split +from _frozen_importlib_external import _os, _path_isfile, _path_isdir, _path_isabs, path_sep, _path_join, _path_split from _io import FileIO import __res as __resource @@ -11,9 +11,9 @@ _b = lambda x: x if isinstance(x, bytes) else utf_8_encode(x)[0] _s = lambda x: x if isinstance(x, str) else utf_8_decode(x)[0] env_entry_point = b'Y_PYTHON_ENTRY_POINT' env_source_root = b'Y_PYTHON_SOURCE_ROOT' -cfg_source_root = b'arcadia-source-root' -env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH' -res_ya_ide_venv = b'YA_IDE_VENV' +cfg_source_root = b'arcadia-source-root' +env_extended_source_search = b'Y_PYTHON_EXTENDED_SOURCE_SEARCH' +res_ya_ide_venv = b'YA_IDE_VENV' executable = sys.executable or 'Y_PYTHON' sys.modules['run_import_hook'] = __resource @@ -21,59 +21,59 @@ sys.modules['run_import_hook'] = __resource py_prefix = b'py/' py_prefix_len = len(py_prefix) -YA_IDE_VENV = __resource.find(res_ya_ide_venv) -Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV - - -def _init_venv(): - if not _path_isabs(executable): - raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable)) - - # Creative copy-paste from site.py - exe_dir, _ = _path_split(executable) - site_prefix, _ = _path_split(exe_dir) - libpath = _path_join(site_prefix, 'lib', - 'python%d.%d' % sys.version_info[:2], - 'site-packages') - sys.path.insert(0, libpath) - - # emulate site.venv() - sys.prefix = site_prefix - sys.exec_prefix = site_prefix - - conf_basename = 'pyvenv.cfg' - candidate_confs = [ - conffile for conffile in ( - _path_join(exe_dir, conf_basename), - _path_join(site_prefix, conf_basename) - ) - if _path_isfile(conffile) - ] - if not candidate_confs: - raise RuntimeError('{} not found'.format(conf_basename)) - virtual_conf = candidate_confs[0] - with FileIO(virtual_conf, 'r') as f: - for line in f: - if b'=' in line: - key, _, value = line.partition(b'=') - key = key.strip().lower() - value = value.strip() - if key == cfg_source_root: - return value - raise RuntimeError('{} key not found in {}'.format(cfg_source_root, virtual_conf)) - - -def _get_source_root(): - env_value = _os.environ.get(env_source_root) - if env_value or not YA_IDE_VENV: - return env_value - - return _init_venv() - - -Y_PYTHON_SOURCE_ROOT = _get_source_root() - - +YA_IDE_VENV = __resource.find(res_ya_ide_venv) +Y_PYTHON_EXTENDED_SOURCE_SEARCH = _os.environ.get(env_extended_source_search) or YA_IDE_VENV + + +def _init_venv(): + if not _path_isabs(executable): + raise RuntimeError('path in sys.executable is not absolute: {}'.format(executable)) + + # Creative copy-paste from site.py + exe_dir, _ = _path_split(executable) + site_prefix, _ = _path_split(exe_dir) + libpath = _path_join(site_prefix, 'lib', + 'python%d.%d' % sys.version_info[:2], + 'site-packages') + sys.path.insert(0, libpath) + + # emulate site.venv() + sys.prefix = site_prefix + sys.exec_prefix = site_prefix + + conf_basename = 'pyvenv.cfg' + candidate_confs = [ + conffile for conffile in ( + _path_join(exe_dir, conf_basename), + _path_join(site_prefix, conf_basename) + ) + if _path_isfile(conffile) + ] + if not candidate_confs: + raise RuntimeError('{} not found'.format(conf_basename)) + virtual_conf = candidate_confs[0] + with FileIO(virtual_conf, 'r') as f: + for line in f: + if b'=' in line: + key, _, value = line.partition(b'=') + key = key.strip().lower() + value = value.strip() + if key == cfg_source_root: + return value + raise RuntimeError('{} key not found in {}'.format(cfg_source_root, virtual_conf)) + + +def _get_source_root(): + env_value = _os.environ.get(env_source_root) + if env_value or not YA_IDE_VENV: + return env_value + + return _init_venv() + + +Y_PYTHON_SOURCE_ROOT = _get_source_root() + + def _print(*xs): """ This is helpful for debugging, since automatic bytes to str conversion is @@ -181,10 +181,10 @@ class ResourceImporter(object): self.source_map = {} # Map from file names to module names. self._source_name = {} # Map from original to altered module names. self._package_prefix = '' - if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH: - self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT)) - else: - self.arcadia_source_finder = None + if Y_PYTHON_SOURCE_ROOT and Y_PYTHON_EXTENDED_SOURCE_SEARCH: + self.arcadia_source_finder = ArcadiaSourceFinder(_s(Y_PYTHON_SOURCE_ROOT)) + else: + self.arcadia_source_finder = None for p in list(self.memory) + list(sys.builtin_module_names): for pp in iter_prefixes(p): @@ -198,13 +198,13 @@ class ResourceImporter(object): importer._package_prefix = name + '.' return importer - def _find_mod_path(self, fullname): - """Find arcadia relative path by module name""" - relpath = resfs_src(mod_path(fullname), resfs_file=True) - if relpath or not self.arcadia_source_finder: - return relpath - return self.arcadia_source_finder.get_module_path(fullname) - + def _find_mod_path(self, fullname): + """Find arcadia relative path by module name""" + relpath = resfs_src(mod_path(fullname), resfs_file=True) + if relpath or not self.arcadia_source_finder: + return relpath + return self.arcadia_source_finder.get_module_path(fullname) + def find_spec(self, fullname, path=None, target=None): try: is_package = self.is_package(fullname) @@ -245,7 +245,7 @@ class ResourceImporter(object): modname = fullname if self.is_package(fullname): fullname += '.__init__' - relpath = self._find_mod_path(fullname) + relpath = self._find_mod_path(fullname) if isinstance(relpath, bytes): relpath = _s(relpath) return relpath or modname @@ -272,7 +272,7 @@ class ResourceImporter(object): fullname += '.__init__' path = mod_path(fullname) - relpath = self._find_mod_path(fullname) + relpath = self._find_mod_path(fullname) if relpath: abspath = resfs_resolve(relpath) if abspath: @@ -298,9 +298,9 @@ class ResourceImporter(object): if fullname + '.__init__' in self.memory: return True - if self.arcadia_source_finder: - return self.arcadia_source_finder.is_package(fullname) - + if self.arcadia_source_finder: + return self.arcadia_source_finder.is_package(fullname) + raise ImportError(fullname) # Extension for contrib/python/coverage. @@ -329,9 +329,9 @@ class ResourceImporter(object): m = rx.match(p) if m: yield prefix + m.group(1), m.group(2) is not None - if self.arcadia_source_finder: - for m in self.arcadia_source_finder.iter_modules(self._package_prefix, prefix): - yield m + if self.arcadia_source_finder: + for m in self.arcadia_source_finder.iter_modules(self._package_prefix, prefix): + yield m def get_resource_reader(self, fullname): try: @@ -394,134 +394,134 @@ class BuiltinSubmoduleImporter(BuiltinImporter): return None -class ArcadiaSourceFinder: - """ - Search modules and packages in arcadia source tree. - See https://wiki.yandex-team.ru/devtools/extended-python-source-search/ for details - """ - NAMESPACE_PREFIX = b'py/namespace/' - PY_EXT = '.py' - YA_MAKE = 'ya.make' - - def __init__(self, source_root): - self.source_root = source_root - self.module_path_cache = {'': set()} - for key, dirty_path in iter_keys(self.NAMESPACE_PREFIX): - # dirty_path contains unique prefix to prevent repeatable keys in the resource storage - path = dirty_path.split(b'/', 1)[1] - namespaces = __resource.find(key).split(b':') - for n in namespaces: - package_name = _s(n.rstrip(b'.')) - self.module_path_cache.setdefault(package_name, set()).add(_s(path)) - # Fill parents with default empty path set if parent doesn't exist in the cache yet - while package_name: - package_name = package_name.rpartition('.')[0] - if package_name in self.module_path_cache: - break - self.module_path_cache.setdefault(package_name, set()) - for package_name in self.module_path_cache.keys(): - self._add_parent_dirs(package_name, visited=set()) - - def get_module_path(self, fullname): - """ - Find file path for module 'fullname'. - For packages caller pass fullname as 'package.__init__'. - Return None if nothing is found. - """ - try: - if not self.is_package(fullname): - return _b(self._cache_module_path(fullname)) - except ImportError: - pass - - def is_package(self, fullname): - """Check if fullname is a package. Raise ImportError if fullname is not found""" - path = self._cache_module_path(fullname) - if isinstance(path, set): - return True - if isinstance(path, str): - return False - raise ImportError(fullname) - - def iter_modules(self, package_prefix, prefix): - paths = self._cache_module_path(package_prefix.rstrip('.')) - if paths is not None: - # Note: it's ok to yield duplicates because pkgutil discards them - - # Yield from cache - import re - rx = re.compile(re.escape(package_prefix) + r'([^.]+)$') - for mod, path in self.module_path_cache.items(): - if path is not None: - m = rx.match(mod) - if m: - yield prefix + m.group(1), self.is_package(mod) - - # Yield from file system - for path in paths: - abs_path = _path_join(self.source_root, path) - for dir_item in _os.listdir(abs_path): - if self._path_is_simple_dir(_path_join(abs_path, dir_item)): - yield prefix + dir_item, True - elif dir_item.endswith(self.PY_EXT) and _path_isfile(_path_join(abs_path, dir_item)): - yield prefix + dir_item[:-len(self.PY_EXT)], False - - def _path_is_simple_dir(self, abs_path): - """ - Check if path is a directory but doesn't contain ya.make file. - We don't want to steal directory from nested project and treat it as a package - """ - return _path_isdir(abs_path) and not _path_isfile(_path_join(abs_path, self.YA_MAKE)) - - def _find_module_in_paths(self, find_package_only, paths, module): - """Auxiliary method. See _cache_module_path() for details""" - if paths: - package_paths = set() - for path in paths: - rel_path = _path_join(path, module) - if not find_package_only: - # Check if file_path is a module - module_path = rel_path + self.PY_EXT - if _path_isfile(_path_join(self.source_root, module_path)): - return module_path - # Check if file_path is a package - if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): - package_paths.add(rel_path) - if package_paths: - return package_paths - - def _cache_module_path(self, fullname, find_package_only=False): - """ - Find module path or package directory paths and save result in the cache - - find_package_only=True - don't try to find module - - Returns: - List of relative package paths - for a package - Relative module path - for a module - None - module or package is not found +class ArcadiaSourceFinder: + """ + Search modules and packages in arcadia source tree. + See https://wiki.yandex-team.ru/devtools/extended-python-source-search/ for details + """ + NAMESPACE_PREFIX = b'py/namespace/' + PY_EXT = '.py' + YA_MAKE = 'ya.make' + + def __init__(self, source_root): + self.source_root = source_root + self.module_path_cache = {'': set()} + for key, dirty_path in iter_keys(self.NAMESPACE_PREFIX): + # dirty_path contains unique prefix to prevent repeatable keys in the resource storage + path = dirty_path.split(b'/', 1)[1] + namespaces = __resource.find(key).split(b':') + for n in namespaces: + package_name = _s(n.rstrip(b'.')) + self.module_path_cache.setdefault(package_name, set()).add(_s(path)) + # Fill parents with default empty path set if parent doesn't exist in the cache yet + while package_name: + package_name = package_name.rpartition('.')[0] + if package_name in self.module_path_cache: + break + self.module_path_cache.setdefault(package_name, set()) + for package_name in self.module_path_cache.keys(): + self._add_parent_dirs(package_name, visited=set()) + + def get_module_path(self, fullname): + """ + Find file path for module 'fullname'. + For packages caller pass fullname as 'package.__init__'. + Return None if nothing is found. + """ + try: + if not self.is_package(fullname): + return _b(self._cache_module_path(fullname)) + except ImportError: + pass + + def is_package(self, fullname): + """Check if fullname is a package. Raise ImportError if fullname is not found""" + path = self._cache_module_path(fullname) + if isinstance(path, set): + return True + if isinstance(path, str): + return False + raise ImportError(fullname) + + def iter_modules(self, package_prefix, prefix): + paths = self._cache_module_path(package_prefix.rstrip('.')) + if paths is not None: + # Note: it's ok to yield duplicates because pkgutil discards them + + # Yield from cache + import re + rx = re.compile(re.escape(package_prefix) + r'([^.]+)$') + for mod, path in self.module_path_cache.items(): + if path is not None: + m = rx.match(mod) + if m: + yield prefix + m.group(1), self.is_package(mod) + + # Yield from file system + for path in paths: + abs_path = _path_join(self.source_root, path) + for dir_item in _os.listdir(abs_path): + if self._path_is_simple_dir(_path_join(abs_path, dir_item)): + yield prefix + dir_item, True + elif dir_item.endswith(self.PY_EXT) and _path_isfile(_path_join(abs_path, dir_item)): + yield prefix + dir_item[:-len(self.PY_EXT)], False + + def _path_is_simple_dir(self, abs_path): + """ + Check if path is a directory but doesn't contain ya.make file. + We don't want to steal directory from nested project and treat it as a package + """ + return _path_isdir(abs_path) and not _path_isfile(_path_join(abs_path, self.YA_MAKE)) + + def _find_module_in_paths(self, find_package_only, paths, module): + """Auxiliary method. See _cache_module_path() for details""" + if paths: + package_paths = set() + for path in paths: + rel_path = _path_join(path, module) + if not find_package_only: + # Check if file_path is a module + module_path = rel_path + self.PY_EXT + if _path_isfile(_path_join(self.source_root, module_path)): + return module_path + # Check if file_path is a package + if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): + package_paths.add(rel_path) + if package_paths: + return package_paths + + def _cache_module_path(self, fullname, find_package_only=False): """ - if fullname not in self.module_path_cache: - parent, _, tail = fullname.rpartition('.') - parent_paths = self._cache_module_path(parent, find_package_only=True) - self.module_path_cache[fullname] = self._find_module_in_paths(find_package_only, parent_paths, tail) - return self.module_path_cache[fullname] - - def _add_parent_dirs(self, package_name, visited): - if not package_name or package_name in visited: - return - visited.add(package_name) - - parent, _, tail = package_name.rpartition('.') - self._add_parent_dirs(parent, visited) - - paths = self.module_path_cache[package_name] - for parent_path in self.module_path_cache[parent]: - rel_path = _path_join(parent_path, tail) - if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): - paths.add(rel_path) - - + Find module path or package directory paths and save result in the cache + + find_package_only=True - don't try to find module + + Returns: + List of relative package paths - for a package + Relative module path - for a module + None - module or package is not found + """ + if fullname not in self.module_path_cache: + parent, _, tail = fullname.rpartition('.') + parent_paths = self._cache_module_path(parent, find_package_only=True) + self.module_path_cache[fullname] = self._find_module_in_paths(find_package_only, parent_paths, tail) + return self.module_path_cache[fullname] + + def _add_parent_dirs(self, package_name, visited): + if not package_name or package_name in visited: + return + visited.add(package_name) + + parent, _, tail = package_name.rpartition('.') + self._add_parent_dirs(parent, visited) + + paths = self.module_path_cache[package_name] + for parent_path in self.module_path_cache[parent]: + rel_path = _path_join(parent_path, tail) + if self._path_is_simple_dir(_path_join(self.source_root, rel_path)): + paths.add(rel_path) + + def excepthook(*args, **kws): # traceback module cannot be imported at module level, because interpreter # is not fully initialized yet @@ -544,19 +544,19 @@ def executable_path_hook(path): raise ImportError(path) -if YA_IDE_VENV: - sys.meta_path.append(importer) - sys.meta_path.append(BuiltinSubmoduleImporter) - if executable not in sys.path: - sys.path.append(executable) - sys.path_hooks.append(executable_path_hook) -else: - sys.meta_path.insert(0, BuiltinSubmoduleImporter) - sys.meta_path.insert(0, importer) - if executable not in sys.path: - sys.path.insert(0, executable) - sys.path_hooks.insert(0, executable_path_hook) - +if YA_IDE_VENV: + sys.meta_path.append(importer) + sys.meta_path.append(BuiltinSubmoduleImporter) + if executable not in sys.path: + sys.path.append(executable) + sys.path_hooks.append(executable_path_hook) +else: + sys.meta_path.insert(0, BuiltinSubmoduleImporter) + sys.meta_path.insert(0, importer) + if executable not in sys.path: + sys.path.insert(0, executable) + sys.path_hooks.insert(0, executable_path_hook) + sys.path_importer_cache[executable] = importer # Indicator that modules and resources are built-in rather than on the file system. diff --git a/library/python/runtime_py3/test/test_arcadia_source_finder.py b/library/python/runtime_py3/test/test_arcadia_source_finder.py index b45f77a910..ff80d0a0a2 100644 --- a/library/python/runtime_py3/test/test_arcadia_source_finder.py +++ b/library/python/runtime_py3/test/test_arcadia_source_finder.py @@ -1,317 +1,317 @@ -import unittest -import yaml -from unittest.mock import patch -from parameterized import parameterized - -import __res as res - - -NAMESPACE_PREFIX = b'py/namespace/' -TEST_SOURCE_ROOT = '/home/arcadia' - - -class ImporterMocks(object): - def __init__(self, mock_fs, mock_resources): - self._mock_fs = mock_fs - self._mock_resources = mock_resources - self._patchers = [ - patch('__res.iter_keys', wraps=self._iter_keys), - patch('__res.__resource.find', wraps=self._resource_find), - patch('__res._path_isdir', wraps=self._path_isdir), - patch('__res._path_isfile', wraps=self._path_isfile), - patch('__res._os.listdir', wraps=self._os_listdir), - ] - for patcher in self._patchers: - patcher.start() - - def stop(self): - for patcher in self._patchers: - patcher.stop() - - def _iter_keys(self, prefix): - assert prefix == NAMESPACE_PREFIX - l = len(prefix) - for k in self._mock_resources.keys(): - yield k, k[l:] - - def _resource_find(self, key): - return self._mock_resources.get(key) - - def _lookup_mock_fs(self, filename): - path = filename.lstrip('/').split('/') - curdir = self._mock_fs - for item in path: - if item in curdir: - curdir = curdir[item] - else: - return None - return curdir - - def _path_isfile(self, filename): - f = self._lookup_mock_fs(filename) - return isinstance(f, str) - - def _path_isdir(self, filename): - f = self._lookup_mock_fs(filename) - return isinstance(f, dict) - - def _os_listdir(self, dirname): - f = self._lookup_mock_fs(dirname) - if isinstance(f, dict): - return f.keys() - else: - return [] - - -class ArcadiaSourceFinderTestCase(unittest.TestCase): - def setUp(self): - self.import_mock = ImporterMocks(yaml.safe_load(self._get_mock_fs()), self._get_mock_resources()) - self.arcadia_source_finder = res.ArcadiaSourceFinder(TEST_SOURCE_ROOT) - - def tearDown(self): - self.import_mock.stop() - - def _get_mock_fs(self): - raise NotImplementedError() - - def _get_mock_resources(self): - raise NotImplementedError() - - -class TestLibraryWithoutNamespace(ArcadiaSourceFinderTestCase): - def _get_mock_fs(self): - return ''' - home: - arcadia: - project: - lib: - mod1.py: "" - package1: - mod2.py: "" - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/project/lib': b'project.lib.', - } - - @parameterized.expand([ - ('project.lib.mod1', b'project/lib/mod1.py'), - ('project.lib.package1.mod2', b'project/lib/package1/mod2.py'), - ('project.lib.unknown_module', None), - ('project.lib', None), # package - ]) - def test_get_module_path(self, module, path): - assert path == self.arcadia_source_finder.get_module_path(module) - - @parameterized.expand([ - ('project.lib.mod1', False), - ('project.lib.package1.mod2', False), - ('project', True), - ('project.lib', True), - ('project.lib.package1', True), - ]) - def test_is_packages(self, module, is_package): - assert is_package == self.arcadia_source_finder.is_package(module) - - def test_is_package_for_unknown_module(self): - self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project.lib.package2')) - - @parameterized.expand([ - ('', { - ('PFX.project', True), - }), - ('project.', { - ('PFX.lib', True), - }), - ('project.lib.', { - ('PFX.mod1', False), - ('PFX.package1', True), - }), - ('project.lib.package1.', { - ('PFX.mod2', False), - }), - ]) - def test_iter_modules(self, package_prefix, expected): - got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') - assert expected == set(got) - - # Check iter_modules() don't crash and return correct result after not existing module was requested - def test_iter_modules_after_unknown_module_import(self): - self.arcadia_source_finder.get_module_path('project.unknown_module') - assert {('lib', True)} == set(self.arcadia_source_finder.iter_modules('project.', '')) - - -class TestLibraryExtendedFromAnotherLibrary(ArcadiaSourceFinderTestCase): - def _get_mock_fs(self): - return ''' - home: - arcadia: - project: - lib: - mod1.py: '' - lib_extension: - mod2.py: '' - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/project/lib': b'project.lib.', - b'py/namespace/unique_prefix2/project/lib_extension': b'project.lib.', - } - - @parameterized.expand([ - ('project.lib.mod1', b'project/lib/mod1.py'), - ('project.lib.mod2', b'project/lib_extension/mod2.py'), - ]) - def test_get_module_path(self, module, path): - assert path == self.arcadia_source_finder.get_module_path(module) - - @parameterized.expand([ - ('project.lib.', { - ('PFX.mod1', False), - ('PFX.mod2', False), - }), - ]) - def test_iter_modules(self, package_prefix, expected): - got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') - assert expected == set(got) - - -class TestNamespaceAndTopLevelLibraries(ArcadiaSourceFinderTestCase): - def _get_mock_fs(self): - return ''' - home: - arcadia: - project: - ns_lib: - mod1.py: '' - top_level_lib: - mod2.py: '' - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/project/ns_lib': b'ns.', - b'py/namespace/unique_prefix2/project/top_level_lib': b'.', - } - - @parameterized.expand([ - ('ns.mod1', b'project/ns_lib/mod1.py'), - ('mod2', b'project/top_level_lib/mod2.py'), - ]) - def test_get_module_path(self, module, path): - assert path == self.arcadia_source_finder.get_module_path(module) - - @parameterized.expand([ - ('ns', True), - ('ns.mod1', False), - ('mod2', False), - ]) - def test_is_packages(self, module, is_package): - assert is_package == self.arcadia_source_finder.is_package(module) - - @parameterized.expand([ - 'project', - 'project.ns_lib', - 'project.top_level_lib', - ]) - def test_is_package_for_unknown_modules(self, module): - self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package(module)) - - @parameterized.expand([ - ('', { - ('PFX.ns', True), - ('PFX.mod2', False), - }), - ('ns.', { - ('PFX.mod1', False), - }), - ]) - def test_iter_modules(self, package_prefix, expected): - got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') - assert expected == set(got) - - -class TestIgnoreDirectoriesWithYaMakeFile(ArcadiaSourceFinderTestCase): - ''' Packages and modules from tests should not be part of pylib namespace ''' - def _get_mock_fs(self): - return ''' - home: - arcadia: - contrib: - python: - pylib: - mod1.py: "" - tests: - conftest.py: "" - ya.make: "" - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/contrib/python/pylib': b'pylib.', - } - - def test_get_module_path_for_lib(self): - assert b'contrib/python/pylib/mod1.py' == self.arcadia_source_finder.get_module_path('pylib.mod1') - - def test_get_module_for_tests(self): - assert self.arcadia_source_finder.get_module_path('pylib.tests.conftest') is None - - def test_is_package_for_tests(self): - self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('pylib.tests')) - - -class TestMergingNamespaceAndDirectoryPackages(ArcadiaSourceFinderTestCase): - ''' Merge parent package (top level in this test) dirs with namespace dirs (DEVTOOLS-8979) ''' - def _get_mock_fs(self): - return ''' - home: - arcadia: - contrib: - python: - pylint: - ya.make: "" - pylint: - __init__.py: "" - patcher: - patch.py: "" - ya.make: "" - ''' - - def _get_mock_resources(self): - return { - b'py/namespace/unique_prefix1/contrib/python/pylint': b'.', - b'py/namespace/unique_prefix1/contrib/python/pylint/patcher': b'pylint.', - } - - @parameterized.expand([ - ('pylint.__init__', b'contrib/python/pylint/pylint/__init__.py'), - ('pylint.patch', b'contrib/python/pylint/patcher/patch.py'), - ]) - def test_get_module_path(self, module, path): - assert path == self.arcadia_source_finder.get_module_path(module) - - -class TestEmptyResources(ArcadiaSourceFinderTestCase): - def _get_mock_fs(self): - return ''' - home: - arcadia: - project: - lib: - mod1.py: '' - ''' - - def _get_mock_resources(self): - return {} - - def test_get_module_path(self): - assert self.arcadia_source_finder.get_module_path('project.lib.mod1') is None - - def test_is_package(self): - self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project')) - - def test_iter_modules(self): - assert [] == list(self.arcadia_source_finder.iter_modules('', 'PFX.')) +import unittest +import yaml +from unittest.mock import patch +from parameterized import parameterized + +import __res as res + + +NAMESPACE_PREFIX = b'py/namespace/' +TEST_SOURCE_ROOT = '/home/arcadia' + + +class ImporterMocks(object): + def __init__(self, mock_fs, mock_resources): + self._mock_fs = mock_fs + self._mock_resources = mock_resources + self._patchers = [ + patch('__res.iter_keys', wraps=self._iter_keys), + patch('__res.__resource.find', wraps=self._resource_find), + patch('__res._path_isdir', wraps=self._path_isdir), + patch('__res._path_isfile', wraps=self._path_isfile), + patch('__res._os.listdir', wraps=self._os_listdir), + ] + for patcher in self._patchers: + patcher.start() + + def stop(self): + for patcher in self._patchers: + patcher.stop() + + def _iter_keys(self, prefix): + assert prefix == NAMESPACE_PREFIX + l = len(prefix) + for k in self._mock_resources.keys(): + yield k, k[l:] + + def _resource_find(self, key): + return self._mock_resources.get(key) + + def _lookup_mock_fs(self, filename): + path = filename.lstrip('/').split('/') + curdir = self._mock_fs + for item in path: + if item in curdir: + curdir = curdir[item] + else: + return None + return curdir + + def _path_isfile(self, filename): + f = self._lookup_mock_fs(filename) + return isinstance(f, str) + + def _path_isdir(self, filename): + f = self._lookup_mock_fs(filename) + return isinstance(f, dict) + + def _os_listdir(self, dirname): + f = self._lookup_mock_fs(dirname) + if isinstance(f, dict): + return f.keys() + else: + return [] + + +class ArcadiaSourceFinderTestCase(unittest.TestCase): + def setUp(self): + self.import_mock = ImporterMocks(yaml.safe_load(self._get_mock_fs()), self._get_mock_resources()) + self.arcadia_source_finder = res.ArcadiaSourceFinder(TEST_SOURCE_ROOT) + + def tearDown(self): + self.import_mock.stop() + + def _get_mock_fs(self): + raise NotImplementedError() + + def _get_mock_resources(self): + raise NotImplementedError() + + +class TestLibraryWithoutNamespace(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: "" + package1: + mod2.py: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/lib': b'project.lib.', + } + + @parameterized.expand([ + ('project.lib.mod1', b'project/lib/mod1.py'), + ('project.lib.package1.mod2', b'project/lib/package1/mod2.py'), + ('project.lib.unknown_module', None), + ('project.lib', None), # package + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('project.lib.mod1', False), + ('project.lib.package1.mod2', False), + ('project', True), + ('project.lib', True), + ('project.lib.package1', True), + ]) + def test_is_packages(self, module, is_package): + assert is_package == self.arcadia_source_finder.is_package(module) + + def test_is_package_for_unknown_module(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project.lib.package2')) + + @parameterized.expand([ + ('', { + ('PFX.project', True), + }), + ('project.', { + ('PFX.lib', True), + }), + ('project.lib.', { + ('PFX.mod1', False), + ('PFX.package1', True), + }), + ('project.lib.package1.', { + ('PFX.mod2', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + # Check iter_modules() don't crash and return correct result after not existing module was requested + def test_iter_modules_after_unknown_module_import(self): + self.arcadia_source_finder.get_module_path('project.unknown_module') + assert {('lib', True)} == set(self.arcadia_source_finder.iter_modules('project.', '')) + + +class TestLibraryExtendedFromAnotherLibrary(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: '' + lib_extension: + mod2.py: '' + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/lib': b'project.lib.', + b'py/namespace/unique_prefix2/project/lib_extension': b'project.lib.', + } + + @parameterized.expand([ + ('project.lib.mod1', b'project/lib/mod1.py'), + ('project.lib.mod2', b'project/lib_extension/mod2.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('project.lib.', { + ('PFX.mod1', False), + ('PFX.mod2', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + +class TestNamespaceAndTopLevelLibraries(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + ns_lib: + mod1.py: '' + top_level_lib: + mod2.py: '' + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/project/ns_lib': b'ns.', + b'py/namespace/unique_prefix2/project/top_level_lib': b'.', + } + + @parameterized.expand([ + ('ns.mod1', b'project/ns_lib/mod1.py'), + ('mod2', b'project/top_level_lib/mod2.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + @parameterized.expand([ + ('ns', True), + ('ns.mod1', False), + ('mod2', False), + ]) + def test_is_packages(self, module, is_package): + assert is_package == self.arcadia_source_finder.is_package(module) + + @parameterized.expand([ + 'project', + 'project.ns_lib', + 'project.top_level_lib', + ]) + def test_is_package_for_unknown_modules(self, module): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package(module)) + + @parameterized.expand([ + ('', { + ('PFX.ns', True), + ('PFX.mod2', False), + }), + ('ns.', { + ('PFX.mod1', False), + }), + ]) + def test_iter_modules(self, package_prefix, expected): + got = self.arcadia_source_finder.iter_modules(package_prefix, 'PFX.') + assert expected == set(got) + + +class TestIgnoreDirectoriesWithYaMakeFile(ArcadiaSourceFinderTestCase): + ''' Packages and modules from tests should not be part of pylib namespace ''' + def _get_mock_fs(self): + return ''' + home: + arcadia: + contrib: + python: + pylib: + mod1.py: "" + tests: + conftest.py: "" + ya.make: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/contrib/python/pylib': b'pylib.', + } + + def test_get_module_path_for_lib(self): + assert b'contrib/python/pylib/mod1.py' == self.arcadia_source_finder.get_module_path('pylib.mod1') + + def test_get_module_for_tests(self): + assert self.arcadia_source_finder.get_module_path('pylib.tests.conftest') is None + + def test_is_package_for_tests(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('pylib.tests')) + + +class TestMergingNamespaceAndDirectoryPackages(ArcadiaSourceFinderTestCase): + ''' Merge parent package (top level in this test) dirs with namespace dirs (DEVTOOLS-8979) ''' + def _get_mock_fs(self): + return ''' + home: + arcadia: + contrib: + python: + pylint: + ya.make: "" + pylint: + __init__.py: "" + patcher: + patch.py: "" + ya.make: "" + ''' + + def _get_mock_resources(self): + return { + b'py/namespace/unique_prefix1/contrib/python/pylint': b'.', + b'py/namespace/unique_prefix1/contrib/python/pylint/patcher': b'pylint.', + } + + @parameterized.expand([ + ('pylint.__init__', b'contrib/python/pylint/pylint/__init__.py'), + ('pylint.patch', b'contrib/python/pylint/patcher/patch.py'), + ]) + def test_get_module_path(self, module, path): + assert path == self.arcadia_source_finder.get_module_path(module) + + +class TestEmptyResources(ArcadiaSourceFinderTestCase): + def _get_mock_fs(self): + return ''' + home: + arcadia: + project: + lib: + mod1.py: '' + ''' + + def _get_mock_resources(self): + return {} + + def test_get_module_path(self): + assert self.arcadia_source_finder.get_module_path('project.lib.mod1') is None + + def test_is_package(self): + self.assertRaises(ImportError, lambda: self.arcadia_source_finder.is_package('project')) + + def test_iter_modules(self): + assert [] == list(self.arcadia_source_finder.iter_modules('', 'PFX.')) diff --git a/library/python/runtime_py3/test/ya.make b/library/python/runtime_py3/test/ya.make index ed0c143080..4ec3db74f5 100644 --- a/library/python/runtime_py3/test/ya.make +++ b/library/python/runtime_py3/test/ya.make @@ -7,11 +7,11 @@ OWNER( DEPENDS(library/python/runtime_py3/test/traceback) -PEERDIR( - contrib/python/parameterized - contrib/python/PyYAML -) - +PEERDIR( + contrib/python/parameterized + contrib/python/PyYAML +) + PY_SRCS( TOP_LEVEL resources/__init__.py @@ -22,7 +22,7 @@ TEST_SRCS( test_metadata.py test_resources.py test_traceback.py - test_arcadia_source_finder.py + test_arcadia_source_finder.py ) RESOURCE_FILES( diff --git a/library/python/testing/yatest_common/ya.make b/library/python/testing/yatest_common/ya.make index 1388ada6f0..5662db4c5d 100644 --- a/library/python/testing/yatest_common/ya.make +++ b/library/python/testing/yatest_common/ya.make @@ -4,8 +4,8 @@ PY23_LIBRARY() OWNER(g:yatest) -NO_EXTENDED_SOURCE_SEARCH() - +NO_EXTENDED_SOURCE_SEARCH() + PY_SRCS( TOP_LEVEL yatest/__init__.py diff --git a/util/ysaveload.h b/util/ysaveload.h index b41922b336..02efb4049b 100644 --- a/util/ysaveload.h +++ b/util/ysaveload.h @@ -421,7 +421,7 @@ public: }; template <class T> -struct TTupleSerializer { +struct TTupleSerializer { template <class F, class Tuple, size_t... Indices> static inline void ReverseUseless(F&& f, Tuple&& t, std::index_sequence<Indices...>) { ApplyToMany( @@ -429,23 +429,23 @@ struct TTupleSerializer { // We need to do this trick because we don't want to break backward compatibility. // Tuples are being packed in reverse order. std::get<std::tuple_size<T>::value - Indices - 1>(std::forward<Tuple>(t))...); - } - + } + static inline void Save(IOutputStream* stream, const T& t) { ReverseUseless([&](const auto& v) { ::Save(stream, v); }, t, std::make_index_sequence<std::tuple_size<T>::value>{}); - } - + } + static inline void Load(IInputStream* stream, T& t) { ReverseUseless([&](auto& v) { ::Load(stream, v); }, t, std::make_index_sequence<std::tuple_size<T>::value>{}); - } -}; - -template <typename... TArgs> + } +}; + +template <typename... TArgs> struct TSerializer<std::tuple<TArgs...>>: TTupleSerializer<std::tuple<TArgs...>> { -}; - +}; + template <> class TSerializer<TBuffer> { public: diff --git a/util/ysaveload_ut.cpp b/util/ysaveload_ut.cpp index 7cffde04f1..723c68f391 100644 --- a/util/ysaveload_ut.cpp +++ b/util/ysaveload_ut.cpp @@ -27,7 +27,7 @@ class TSaveLoadTest: public TTestBase { UNIT_TEST(TestNewStyle) UNIT_TEST(TestNewNewStyle) UNIT_TEST(TestList) - UNIT_TEST(TestTuple) + UNIT_TEST(TestTuple) UNIT_TEST(TestVariant) UNIT_TEST(TestInheritNonVirtualClass) UNIT_TEST(TestInheritVirtualClass) @@ -390,22 +390,22 @@ private: UNIT_ASSERT_VALUES_EQUAL(*std::next(list.begin(), 1), 1); UNIT_ASSERT_VALUES_EQUAL(*std::next(list.begin(), 2), 10); } - - void TestTuple() { - TBufferStream s; - - using TTuple = std::tuple<int, TString, unsigned int>; - const TTuple toSave{-10, "qwerty", 15}; - Save(&s, toSave); - - TTuple toLoad; - Load(&s, toLoad); - - UNIT_ASSERT_VALUES_EQUAL(std::get<0>(toLoad), std::get<0>(toSave)); - UNIT_ASSERT_VALUES_EQUAL(std::get<1>(toLoad), std::get<1>(toSave)); - UNIT_ASSERT_VALUES_EQUAL(std::get<2>(toLoad), std::get<2>(toSave)); - } - + + void TestTuple() { + TBufferStream s; + + using TTuple = std::tuple<int, TString, unsigned int>; + const TTuple toSave{-10, "qwerty", 15}; + Save(&s, toSave); + + TTuple toLoad; + Load(&s, toLoad); + + UNIT_ASSERT_VALUES_EQUAL(std::get<0>(toLoad), std::get<0>(toSave)); + UNIT_ASSERT_VALUES_EQUAL(std::get<1>(toLoad), std::get<1>(toSave)); + UNIT_ASSERT_VALUES_EQUAL(std::get<2>(toLoad), std::get<2>(toSave)); + } + template <class TVariant, class T> void TestVariantImpl(TVariant& v, const T& expected) { v = expected; |