path: root/build/conf/python.conf

# tag:python-specific tag:cython-specific
CYTHON_SCRIPT=${input:"${ARCADIA_ROOT}/contrib/tools/cython/cython.py"} ${hide:CYTHON_FAKE_ID} $CYTHON_PREDEFINED_INCLUDES
RUN_CYTHON_SCRIPT=$YMAKE_PYTHON $CYTHON_SCRIPT
RUN_CYTHON_SCRIPT_H=$YMAKE_PYTHON $CYTHON_SCRIPT
PYTHON_TYPE_FOR_CYTHON=PY2


CYTHON_EMBEDED_FILES=\
contrib/tools/cython/Cython/Utility/Buffer.c \
contrib/tools/cython/Cython/Utility/Builtins.c \
contrib/tools/cython/Cython/Utility/CMath.c \
contrib/tools/cython/Cython/Utility/Capsule.c \
contrib/tools/cython/Cython/Utility/CommonTypes.c \
contrib/tools/cython/Cython/Utility/Complex.c \
contrib/tools/cython/Cython/Utility/Coroutine.c \
contrib/tools/cython/Cython/Utility/CythonFunction.c \
contrib/tools/cython/Cython/Utility/Embed.c \
contrib/tools/cython/Cython/Utility/Exceptions.c \
contrib/tools/cython/Cython/Utility/ExtensionTypes.c \
contrib/tools/cython/Cython/Utility/FunctionArguments.c \
contrib/tools/cython/Cython/Utility/ImportExport.c \
contrib/tools/cython/Cython/Utility/MemoryView_C.c \
contrib/tools/cython/Cython/Utility/ModuleSetupCode.c \
contrib/tools/cython/Cython/Utility/ObjectHandling.c \
contrib/tools/cython/Cython/Utility/Optimize.c \
contrib/tools/cython/Cython/Utility/Overflow.c \
contrib/tools/cython/Cython/Utility/Printing.c \
contrib/tools/cython/Cython/Utility/Profile.c \
contrib/tools/cython/Cython/Utility/StringTools.c \
contrib/tools/cython/Cython/Utility/TestUtilityLoader.c \
contrib/tools/cython/Cython/Utility/TypeConversion.c

macro _UNWRAP_TEXT_INPUTS(Files...) {
    .CMD=${hide;context=TEXT;input:Files}
}

macro _UNWRAP_OUTPUT_INCLUDES(Files...) {
    .CMD=${hide;output_include:Files}
}


CYTHON_OUTPUT_INCLUDES=\
${hide;output_include:"contrib/libs/python/Include/compile.h"} \
${hide;output_include:"contrib/libs/python/Include/frameobject.h"} \
${hide;output_include:"contrib/libs/python/Include/longintrepr.h"} \
${hide;output_include:"contrib/libs/python/Include/pyconfig.h"} \
${hide;output_include:"contrib/libs/python/Include/Python.h"} \
${hide;output_include:"contrib/libs/python/Include/pythread.h"} \
${hide;output_include:"contrib/libs/python/Include/structmember.h"} \
${hide;output_include:"contrib/libs/python/Include/traceback.h"} \
${hide;output_include:"contrib/tools/cython/generated_c_headers.h"} \
${hide;output_include:"omp.h"} \
$_UNWRAP_OUTPUT_INCLUDES($CYTHON_EMBEDED_FILES)

# tag:cython-specific
CYTHON_CPP_OUTPUT_INCLUDES=${hide;output_include:"contrib/tools/cython/generated_cpp_headers.h"}

# tag:cython-specific
CYTHON_PREDEFINED_INCLUDES=$_UNWRAP_TEXT_INPUTS($CYTHON_EMBEDED_FILES)

# tag:cython-specific
CYTHON_PREDEFINED_INDUCED=\
    contrib/libs/python/Include/Python.h

# tag:cython-specific
CYTHON_OPTIONS=
when ($OS_LINUX) {
    CYTHON_OPTIONS += -E UNAME_SYSNAME=Linux
}
elsewhen ($OS_DARWIN) {
    CYTHON_OPTIONS += -E UNAME_SYSNAME=Darwin
}
elsewhen ($OS_WINDOWS) {
    CYTHON_OPTIONS += -E UNAME_SYSNAME=Windows
}

# tag:python-specific
# PYTHON_DISTR={ "default" | "macos-brew" }  determine python system supply
# see build/platform/python/ya.make
PYTHON_DISTR="default"
when ($USE_SYSTEM_PYTHON) {
    USE_ARCADIA_PYTHON=no
    select ($USE_SYSTEM_PYTHON) {
        "2.7" ? {
            _SYSTEM_PYTHON27=yes
            PY_VERSION=2.7
            PY_FRAMEWORK_VERSION=2.7
        }
        "3.4" ? {
            _SYSTEM_PYTHON34=yes
            PY_VERSION=3.4m
            PY_FRAMEWORK_VERSION=3.4
        }
        "3.5" ? {
            _SYSTEM_PYTHON35=yes
            PY_VERSION=3.5m
            PY_FRAMEWORK_VERSION=3.5
        }
        "3.6" ? {
            _SYSTEM_PYTHON36=yes
            PY_VERSION=3.6m
            PY_FRAMEWORK_VERSION=3.6
        }
        "3.7" ? {
            _SYSTEM_PYTHON37=yes
            PY_VERSION=3.7m
            PY_FRAMEWORK_VERSION=3.7
        }
        "3.8" ? {
            _SYSTEM_PYTHON38=yes
            PY_VERSION=3.8
            PY_FRAMEWORK_VERSION=3.8
        }
        "3.9" ? {
            _SYSTEM_PYTHON39=yes
            PY_VERSION=3.9
            PY_FRAMEWORK_VERSION=3.9
        }
        "3.10" ? {
            _SYSTEM_PYTHON310=yes
            PY_VERSION=3.10
            PY_FRAMEWORK_VERSION=3.10
        }
        "3.11" ? {
            _SYSTEM_PYTHON311=yes
            PY_VERSION=3.11
            PY_FRAMEWORK_VERSION=3.11
        }
        "3.12" ? {
            _SYSTEM_PYTHON312=yes
            PY_VERSION=3.12
            PY_FRAMEWORK_VERSION=3.12
        }
        default ? {
            FATAL_ERROR_MESSAGE+=invalid USE_SYSTEM_PYTHON value
        }
    }
}

when ($BUILD_PYTHON3_BIN) {
    YMAKE_PYTHON3=${BUILD_PYTHON3_BIN}
    YMAKE_PYTHON3_PEERDIR=
}
otherwise {
    YMAKE_PYTHON3=${YMAKE_PYTHON3_RESOURCE_GLOBAL}/bin/python3
}

# tag:python-specific
when ($USE_ARCADIA_PYTHON == "no") {
    SYSINCL+=build/sysincl/python.yml
}

when ($OPENSOURCE) {
    SYSINCL+=build/sysincl/python-2-disable.yml
}

# tag:python-specific
PYTHON2=no
PYTHON3=no

# tag:python-specific
_PYTHON_VER=py2

# tag:python-specific
PYTHON_IMPORT_TRACING=yes

# tag:python-specific
when (!$ARCADIA_PYTHON_UNICODE_SIZE) {
    when ($OS_WINDOWS) {
        ARCADIA_PYTHON_UNICODE_SIZE=2
    }
    otherwise {
        ARCADIA_PYTHON_UNICODE_SIZE=4
    }
}

YMAKE_PYTHON3_PEERDIR=build/platform/python/ymake_python3
macro NO_YMAKE_PYTHON3() {
    SET(YMAKE_PYTHON3_PEERDIR)
}

PYTHON_LINTERS_DEFAULT_CONFIGS=build/config/tests/py_style/default_configs.json

FLAKE_EXTRA_PARAMS=
FLAKE_MIGRATIONS_CONFIG=
when ($DISABLE_FLAKE8_MIGRATIONS == "yes") {
    FLAKE_EXTRA_PARAMS="DISABLE_FLAKE8_MIGRATIONS=yes"
}
otherwise {
    FLAKE_MIGRATIONS_CONFIG=build/rules/flake8/migrations.yaml
}

when ($FLAKE8_FILE_PROCESSING_TIME == "") {
    FLAKE8_FILE_PROCESSING_TIME=1.5
}

when ($BLACK_FILE_PROCESSING_TIME == "") {
    BLACK_FILE_PROCESSING_TIME=1.5
}

# tag:python-specific tag:deprecated tag:internal
module _PY2_PROGRAM: _BASE_PY_PROGRAM {
    SET(MODULE_LANG PY2)

    # Looks like we cannot avoid copy-paste util ymake supports multiple inheritance
    # We need to attach coverage.extractor to every py_program target, except pytest targets
    ADD_YTEST($MODULE_PREFIX$REALPRJNAME coverage.extractor)
    STYLE_PY2_FLAKE8()
}

# tag:python-specific tag:deprecated
### @usage: PY2_PROGRAM([progname]) # deprecated
###
### Deprecated. Use PY3_PROGRAM instead.
### Python 2.x binary program. Links all Python 2.x libraries and Python 2.x interpreter into itself to form regular executable.
### If name is not specified it will be generated from the name of the containing project directory.
### This only compatible with PYTHON2-tagged modules and selects those from multimodules.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
module PY2_PROGRAM: _PY2_PROGRAM {
    .RESTRICTED=STYLE_PYTHON
    when ($FAIL_PY2 == "yes") {
        _OK=no
    }
    elsewhen ($OPENSOURCE != "yes") {
        PEERDIR+=build/rules/py2_deprecation
    }
    ASSERT(_OK You are using deprecated Python2-only code (PY2_PROGRAM). Please consider rewriting to Python 3.)
}

# tag:python-specific
### @usage: NO_EXTENDED_SOURCE_SEARCH()
###
### Prevent module using in extended python source search.
### Use the macro if module contains python2-only files (or other python sources which shouldn't be imported by python3 interpreter)
### which resides in the same directories with python 3 useful code. contrib/python/future is a example.
### Anyway, preferred way is to move such files into separate dir and don't use this macro at all.
###
### Also see: https://docs.yandex-team.ru/ya-make/manual/python/vars#y_python_extended_source_search for details
macro NO_EXTENDED_SOURCE_SEARCH() {
    SET(NO_EXTENDED_SOURCE_SEARCH yes)
}

# tag:python-specific
### @usage: PY3_PROGRAM([progname])
###
### Python 3.x binary program. Links all Python 3.x libraries and Python 3.x interpreter into itself to form regular executable.
### If name is not specified it will be generated from the name of the containing project directory.
### This only compatible with PYTHON3-tagged modules and selects those from multimodules.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
multimodule PY3_PROGRAM {
    module PY3_BIN: PY3_PROGRAM_BIN {
        .PEERDIRSELF=PY3_BIN_LIB
        .IGNORED=RESOURCE RESOURCE_FILES PEERDIR TASKLET_REG RUN_ANTLR4_PYTHON BUILDWITH_CYTHON_CPP _BUILDWITH_CYTHON_CPP_DEP _BUILDWITH_CYTHON_CPP_H BUILDWITH_CYTHON_C _BUILDWITH_CYTHON_C_DEP _BUILDWITH_CYTHON_C_H _BUILDWITH_CYTHON_C_API_H
        # Notify pybuild to skip almost all PY_SRCS arguments except MAIN
        ENABLE(PROCESS_PY_MAIN_ONLY)
        _PY_PROGRAM()
    }

    module PY3_BIN_LIB: PY3_LIBRARY {
        .IGNORED=LICENSE_RESTRICTION LICENSE_RESTRICTION_EXCEPTIONS
        # Notify pybuild to skip all python main function definitions
        ENABLE(IGNORE_PY_MAIN)
        NO_CLANG_TIDY()
    }
}

# tag:python-specific tag:internal
### @usage: _ADD_PY_LINTER_CHECK(NAME name LINTER linter [DEPENDS deps] CONFIGS configs_file [GLOBAL_RESOURCES gr] [FILE_PROCESSING_TIME fpt] [EXTRA_PARAMS params] [PROJECT_TO_CONFIG_MAP pcm] [FLAKE_MIGRATIONS_CONFIG fmc] [CUSTOM_CONFIG cc] [CONFIG_TYPE ct])
###
### Triggers respective plugin
macro _ADD_PY_LINTER_CHECK(Args...) {
    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/${PYTHON_LINTERS_DEFAULT_CONFIGS})
}

# tag:python-specific tag:internal
### @usage: STYLE_DUMMY()
###
### Not an actual linter, used for dummy linter demonstration
macro STYLE_DUMMY() {
    _ADD_PY_LINTER_CHECK(NAME dummy_linter LINTER tools/dummy_linter/dummy_linter CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS)
}

# tag:python-specific tag:test
### @usage: STYLE_PYTHON([CONFIG_TYPE config_type])
###
### Check python3 sources for style issues using black.
macro STYLE_PYTHON(CONFIG_TYPE="") {
    .ALLOWED_IN_COMMON=yes
    _ADD_PY_LINTER_CHECK(NAME black LINTER tools/black_linter/black_linter FILE_PROCESSING_TIME $BLACK_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS CUSTOM_CONFIG $CONFIG_TYPE CONFIG_TYPE $CONFIG_TYPE)
}

# tag:python-specific tag:test
### @usage: STYLE_RUFF([CONFIG_TYPE config_type])
###
### Check python3 sources for style issues using ruff.
RUFF_PROJECT_TO_CONFIG_MAP=build/config/tests/ruff/ruff_config_paths.json
macro STYLE_RUFF(CONFIG_TYPE="") {
    .ALLOWED_IN_COMMON=yes
    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/${RUFF_PROJECT_TO_CONFIG_MAP})
    _ADD_PY_LINTER_CHECK(NAME ruff LINTER tools/ruff_linter/bin/ruff_linter GLOBAL_RESOURCES build/external_resources/ruff CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS PROJECT_TO_CONFIG_MAP $RUFF_PROJECT_TO_CONFIG_MAP CONFIG_TYPE $CONFIG_TYPE)
}

# tag:python-specific tag:test
### @usage: STYLE_FLAKE8()
###
### Check python3 sources for style issues using flake8.
macro STYLE_FLAKE8() {
    _ADD_PY_LINTER_CHECK(NAME flake8 LINTER tools/flake8_linter/flake8_linter GLOBAL_RESOURCES build/external_resources/flake8_py3 FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS FLAKE_MIGRATIONS_CONFIG $FLAKE_MIGRATIONS_CONFIG EXTRA_PARAMS $FLAKE_EXTRA_PARAMS)
}

# tag:python-specific tag:test
### @usage: STYLE_PY2_FLAKE8()
###
### Check python3 sources for style issues using flake8.
macro STYLE_PY2_FLAKE8() {
    _ADD_PY_LINTER_CHECK(NAME py2_flake8 LINTER tools/flake8_linter/flake8_linter GLOBAL_RESOURCES build/external_resources/flake8_py2 FILE_PROCESSING_TIME $FLAKE8_FILE_PROCESSING_TIME CONFIGS $PYTHON_LINTERS_DEFAULT_CONFIGS FLAKE_MIGRATIONS_CONFIG $FLAKE_MIGRATIONS_CONFIG EXTRA_PARAMS $FLAKE_EXTRA_PARAMS)
}

# tag:python-specific tag:test
### @usage: NO_DOCTESTS()
###
### Disable doctests in PY[|3|23_]TEST
macro NO_DOCTESTS() {
    ENV(YA_PYTEST_DISABLE_DOCTEST="yes")
}

# tag:python-specific tag:test
### @usage: _BASE_PYTEST  # internal
###
### Base logic of Python 2.x py.test modules: common module properties and dependencies.
module _BASE_PYTEST: _BASE_PY_PROGRAM {
    .NODE_TYPE=Program
    .FINAL_TARGET=no
    .ALLOWED=YT_SPEC NO_DOCTESTS
    when ($USE_ARCADIA_PYTHON == "yes") {
        PEERDIR+=library/python/pytest
    }
    when ($USE_ARCADIA_PYTHON == "no") {
        MODULE_SUFFIX=.pkg.fake
        PEERDIR+=library/python/pytest/empty
    }
}

# tag:python-specific tag:deprecated tag:test
### @usage: PYTEST_BIN() #deprecated
###
### Same as PY2TEST. Don't use this, use PY2TEST instead.
module PYTEST_BIN: _BASE_PYTEST {
    .NODE_TYPE=Program
    .DEFAULT_NAME_GENERATOR=FullPath
    .ARGS_PARSER=Base
    SETUP_PYTEST_BIN()
    STYLE_PY2_FLAKE8()
}

# tag:python-specific tag:test
### @usage: PY2TEST([name])
###
### The test module for Python 2.x based on py.test
###
### This module is compatible only with PYTHON2-tagged modules and selects peers from multimodules accordingly.
### This module is compatible with non-Arcadia Python builds.
###
### Documentation: https://wiki.yandex-team.ru/yatool/test/#python
### Documentation about the Arcadia test system: https://wiki.yandex-team.ru/yatool/test/
module PY2TEST: PYTEST_BIN {
    .RESTRICTED=STYLE_PYTHON
    .DEFAULT_NAME_GENERATOR=FullPath
    .ARGS_PARSER=Base
    when ($FAIL_PY2 == "yes") {
        _OK=no
    }
    elsewhen ($OPENSOURCE != "yes") {
        PEERDIR+=build/rules/py2_deprecation
    }
    SET(MODULE_LANG PY2)
    ASSERT(_OK You are using deprecated Python2-only code (PY2TEST). Please consider rewriting to Python 3.)

    SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS canondata/result.json)
    NO_CLANG_TIDY()
    _DONT_REQUIRE_LICENSE()
    WITHOUT_VERSION()
}

# tag:python-specific tag:deprecated tag:test
### @usage: PY3TEST_BIN() #deprecated
###
### Same as PY3TEST. Don't use this, use PY3TEST instead.
module PY3TEST_BIN: _BASE_PY3_PROGRAM {
    .NODE_TYPE=Program
    .FINAL_TARGET=no
    .ALLOWED=YT_SPEC NO_DOCTESTS
    .DEFAULT_NAME_GENERATOR=FullPath
    .ARGS_PARSER=Base
    SET(MODULE_LANG PY3)
    SETUP_PYTEST_BIN()
    STYLE_FLAKE8()
    PEERDIR+=library/python/pytest
}

# tag:python-specific tag:test
### Disable submodules with the specified name(s).
### Unlike EXCLUDE_TAGS this variable:
### - affects all projects in the build;
### - filter by module name and ignore MODULE_TAG.
EXCLUDE_SUBMODULES=PY3TEST_LIBRARY

# tag:python-specific tag:test
### @usage: PY3TEST([name])
###
### The test module for Python 3.x based on py.test
###
### This module is compatible only with PYTHON3-tagged modules and selects peers from multimodules accordingly.
### This module is only compatible with Arcadia Python build (to avoid tests duplication from Python2/3-tests). For non-Arcadia python use PYTEST.
###
### Documentation: https://wiki.yandex-team.ru/yatool/test/#testynapytest
### Documentation about the Arcadia test system: https://wiki.yandex-team.ru/yatool/test/
multimodule PY3TEST {
    module PY3TEST_PROGRAM: PY3TEST_BIN {
        .IGNORED=RUN_ANTLR4_PYTHON
        .FINAL_TARGET=yes

        SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS canondata/result.json)
        _DONT_REQUIRE_LICENSE()
        WITHOUT_VERSION()
    }

    module PY3TEST_LIBRARY: PY3_LIBRARY {
        PEERDIR+=library/python/pytest
        _DONT_REQUIRE_LICENSE()
        WITHOUT_VERSION()
    }
}

# tag:python-specific
### @usage: PY_ANY_MODULE(name major_ver [minor_ver] [EXPORTS symlist_file] [PREFIX prefix])
###
### The Python external module for any versio of Arcadia or system Python.
### 1. major_ver and minor_ver must be integers.
### 2. The resulting .so will have the prefix "lib".
### 3. Processing EXPORTS and PREFIX is the same as for DLL module
### This is native DLL, so it will select C++ version from PROTO_LIBRARY.
###
### Note: Use PYTHON2_MODULE()/PYTHON3_MODULE() in order to PEERDIR proper version of PY23_NATIVE_LIBRARY.
### Do not PEERDIR any PY*_LIBRARY: this will link Python in and render artifact unusable as Python module.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
module PY_ANY_MODULE: DLL_UNIT {
    .SEM=CPP_DYN_LIBRARY_SEM
    when ($MSVC != "yes" && $DARWIN != "yes") {
        LDFLAGS+= -Wl,-Bsymbolic
    }

    when ($USE_ARCADIA_PYTHON == "no") {
        when ($USE_SYSTEM_PYTHON) {
            PEERDIR+=build/platform/python
        }
        otherwise {
            when ($MSVC == "yes" || $CYGWIN == "yes") {
                LDFLAGS+=$PYTHON_LIBRARIES
            }
        }
    }

    # This by now replicates ymake's behavior. We'll get rid of SONAME setting in ymake and fix this code altogether
    SONAME=${pre=$MODULE_PREFIX;suf=$MODULE_SUFFIX:REALPRJNAME}
    when ($MODULE_VERSION) {
       LINK_DYN_LIB_FLAGS=--soname ${output;pre=$MODULE_PREFIX;suf=$MODULE_SUFFIX$MODULE_VERSION:REALPRJNAME}
    }
    _SONAME=$SONAME$MODULE_VERSION

    # -bundle
    when ($DARWIN == "yes") {
        when ($CLANG_VER == "16") {
            # TODO(pg): also check 17 and beyond
        }
        otherwise {
            LDFLAGS+=-flat_namespace
        }
    }

    when ($MSVC == "yes" || $CYGWIN == "yes") {
        MODULE_PREFIX=
        MODULE_SUFFIX=.pyd
    }
    otherwise {
        MODULE_PREFIX=
        MODULE_SUFFIX=.so
    }
    .RESTRICTED=USE_PYTHON2 USE_PYTHON3 PY_SRCS PY_MAIN
}

# tag:python-specific
### @usage: PY2MODULE(name major_ver [minor_ver] [EXPORTS symlist_file] [PREFIX prefix])
###
### The Python external module for Python2 and any system Python
### 1. major_ver and minor_ver must be integers.
### 2. The resulting .so will have the prefix "lib".
### 3. Processing EXPORTS and PREFIX is the same as for DLL module
### This is native DLL, so it will select C++ version from PROTO_LIBRARY.
###
### Note: this module will always PEERDIR Python2 version of PY23_NATIVE_LIBRARY.
### Do not PEERDIR PY2_LIBRARY or PY23_LIBRARY: this will link Python in and render artifact unusable as Python module.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
module PY2MODULE: PY_ANY_MODULE {
    PYTHON2_MODULE()
    SET(MODULE_LANG PY2)
    .RESTRICTED=PYTHON3_MODULE PYTHON3_ADDINCL
}

# tag:python-specific
### @usage: PY3MODULE(name major_ver [minor_ver] [EXPORTS symlist_file] [PREFIX prefix])
###
### The Python external module for Python3 and any system Python
### 1. major_ver and minor_ver must be integers.
### 2. The resulting .so will have the prefix "lib".
### 3. Processing EXPORTS and PREFIX is the same as for DLL module
### This is native DLL, so it will select C++ version from PROTO_LIBRARY.
###
### Note: this module will always PEERDIR Python3 version of PY23_NATIVE_LIBRARY.
### Do not PEERDIR PY3_LIBRARY or PY23_LIBRARY: this will link Python in and render artifact unusable as Python module.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
module PY3MODULE: PY_ANY_MODULE {
    PYTHON3_MODULE()
    SET(MODULE_LANG PY3)
    .RESTRICTED=PYTHON2_MODULE PYTHON2_ADDINCL
}

# tag:python-specific
### @usage: PYTHON2_MODULE()
###
### Use in PY_ANY_MODULE to set it up for Python 2.x.
macro PYTHON2_MODULE() {
    when ($USE_ARCADIA_PYTHON == "yes" && $MSVC == "yes" || $IS_CROSS_TOOLS == "yes") {
        PEERDIR+=contrib/tools/python/lib
    }
    SET(ANTLR_PYTHON Python2)
    PYTHON2_ADDINCL()
}

# tag:python-specific
### @usage: PYTHON3_MODULE()
###
### Use in PY_ANY_MODULE to set it up for Python 3.x.
macro PYTHON3_MODULE() {
    when ($USE_ARCADIA_PYTHON == "yes" && $MSVC == "yes" || $IS_CROSS_TOOLS == "yes") {
        PEERDIR+=contrib/tools/python3/lib2
    }

    SET(ANTLR_PYTHON Python3)
    PYTHON3_ADDINCL()
}

# tag:python-specific
module _PY_PACKAGE: UNION {
    .EXTS=.py
    .ALLOWED=GRPC USE_SKIFF
    .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER
    ENABLE(PY_PROTOS_FOR)
    SET(PEERDIR_TAGS PY_PROTO PY_PROTO_FROM_SCHEMA)
    SET(DONT_RESOLVE_INCLUDES no)
}

# tag:python-specific tag:deprecated tag:internal
### @usage: PY_PACKAGE(name) # internal, deprecated
###
### This is module created via PY_PROTOS_FOR() macro
module PY_PACKAGE: _PY_PACKAGE {
    .ALIASES=SRCS=SRCS
    .FINAL_TARGET=yes
}

# tag:python-specific
PY_NAMESPACE_VALUE=

# tag:python-specific
### @usage: PY_NAMESPACE(prefix)
###
### Sets default Python namespace for all python sources in the module.
### Especially suitable in PROTO_LIBRARY where Python sources are generated and there is no PY_SRCS to place NAMESPACE parameter.
macro PY_NAMESPACE(Arg) {
    SET(PY_NAMESPACE_VALUE $Arg)
}

# tag:python-specific tag:deprecated
### @usage: PY2_LIBRARY() # deprecated
###
### Deprecated. Use PY23_LIBRARY or PY3_LIBRARY instead.
### Python 2.x binary built library. Builds sources from PY_SRCS to data suitable for PY2_PROGRAM.
### Adds dependencies to Python 2.x runtime library from Arcadia.
### This module is only compatible with PYTHON2-tagged modules and selects those from multimodules.
### This module is only compatible with Arcadia Python build.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
module PY2_LIBRARY: _LIBRARY {
    _ARCADIA_PYTHON_ADDINCL()

    when ($NO_PYTHON_INCLS != "yes") {
        PEERDIR+=contrib/libs/python
    }
    when ($MSVC == "yes" || $CYGWIN == "yes") {
        MODULE_PREFIX=py
    }
    otherwise {
        MODULE_PREFIX=libpy
    }
    .ALLOWED=FORK_TESTS
    .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_PYTHON2
    .RESTRICTED=PYTHON3_ADDINCL USE_PYTHON3 PYTHON2_ADDINCL OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS STYLE_PYTHON
    USE_GLOBAL_CMD=yes

    when ($PY_PROTO_MYPY_ENABLED == "yes") {
        PY_PROTO_MYPY_SUFFIX=_pb2.pyi
        PY_PROTO_MYPY_PLUGIN=$PY_PROTO_MYPY_PLUGIN_BASE ${hide;noauto;output;norel;nopath;noext;suf=_pb2.pyi:File}
        PY_PROTO_MYPY_PLUGIN_INTERNAL=$PY_PROTO_MYPY_PLUGIN_BASE ${hide;noauto;output;norel;nopath;noext;suf=__intpy2___pb2.pyi:File} ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__intpy2___pb2.pyi:File} ${nopath;noext;suf=_pb2.pyi:File}"})
    }
    SET(MODULE_LANG PY2)

    STYLE_PY2_FLAKE8()
    ADD_CLANG_TIDY()
    when ($TIDY_ENABLED == "yes") {
        _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json
        _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_default_map.json
    }
}


# tag:python-specific
### @usage: PY3_LIBRARY()
###
### Python 3.x binary library. Builds sources from PY_SRCS to data suitable for PY2_PROGRAM
### Adds dependencies to Python 2.x runtime library from Arcadia.
### This module is only compatible with PYTHON3-tagged modules and selects those from multimodules.
### This module is only compatible with Arcadia Python build.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
module PY3_LIBRARY: _LIBRARY {
    _ARCADIA_PYTHON3_ADDINCL()

    when ($NO_PYTHON_INCLS != "yes") {
        PEERDIR+=contrib/libs/python
    }
    when ($MSVC == "yes" || $CYGWIN == "yes") {
        MODULE_PREFIX=py3
    }
    otherwise {
        MODULE_PREFIX=libpy3
    }
    .ALLOWED=FORK_TESTS
    .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER
    .RESTRICTED=PYTHON3_ADDINCL USE_PYTHON3 PYTHON2_ADDINCL USE_PYTHON2 OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS
    USE_GLOBAL_CMD=yes

    RUN_CYTHON_SCRIPT=$YMAKE_PYTHON3 $CYTHON_SCRIPT
    RUN_CYTHON_SCRIPT_H=$YMAKE_PYTHON3 $CYTHON_SCRIPT
    PYTHON_TYPE_FOR_CYTHON=PY3
    ANTLR_PYTHON=Python3

    when ($PY_PROTO_MYPY_ENABLED == "yes") {
        PY_PROTO_MYPY_SUFFIX=_pb2.pyi
        PY_PROTO_MYPY_PLUGIN=$PY_PROTO_MYPY_PLUGIN_BASE ${hide;noauto;output;norel;nopath;noext;suf=_pb2.pyi:File}
        PY_PROTO_MYPY_PLUGIN_INTERNAL=$PY_PROTO_MYPY_PLUGIN_BASE ${hide;noauto;output;norel;nopath;noext;suf=__intpy3___pb2.pyi:File} ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__intpy3___pb2.pyi:File} ${nopath;noext;suf=_pb2.pyi:File}"})
    }
    SET(MODULE_LANG PY3)

    STYLE_FLAKE8()
    ADD_CLANG_TIDY()
    when ($TIDY_ENABLED == "yes") {
        _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json
        _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_default_map.json
    }
}


# tag:python-specific
### @usage: _BASE_PY_PROGRAM #internal
###
### The base module for all Python 2.x binary programs. Adds linking logic, relevant module properties and
### dependency on Python 2.x interpreter. Also adds import tests on all sources including PEERDIR'ed libraries.
### Links all Python 2.x libraries and Python 2.x interpreter into itself to form regular executable.
### This only compatible with PYTHON2-tagged modules and selects those from multimodules.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
module _BASE_PY_PROGRAM: _BASE_PROGRAM {
    .CMD=$PY_PROGRAM_LINK_EXE
    _ARCADIA_PYTHON_ADDINCL()

    when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") {
        NO_CHECK_IMPORTS_FOR_VALUE=
    }
    ADD_CHECK_PY_IMPORTS()

    when ($NO_PYTHON_INCLS != "yes") {
        PEERDIR += contrib/libs/python
    }
    when ($USE_ARCADIA_PYTHON == "yes") {
        PEERDIR += library/python/runtime/main
        PEERDIR += contrib/deprecated/python/subprocess32
        when ($PYTHON_SQLITE3 != "no") {
            PEERDIR += contrib/tools/python/src/Modules/_sqlite
        }
    }
    when ($PYTHON_COVERAGE == "yes") {
        PEERDIR+=library/python/coverage
    }

    when ($PYTHON_IMPORT_TRACING == "yes") {
        PEERDIR += library/python/import_tracing/constructor
    }

    when ($ARCH_PPC64LE == "yes") {
        _MY_ALLOCATOR=SYSTEM
    }
    otherwise {
        _MY_ALLOCATOR=J
    }
    ALLOCATOR($_MY_ALLOCATOR)
    STRIP()

    when ($BUILD_TYPE == "DEBUG" || $BUILD_TYPE == "FASTDEBUG") {
        NO_STRIP=yes
    }
    when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") {
        NO_STRIP=yes
    }
    .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_PYTHON2
    .RESTRICTED=PYTHON3_ADDINCL USE_PYTHON3 PYTHON2_ADDINCL OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS
    SET(MODULE_LANG PY2)
}

# tag:python-specific tag:codenav
when ($CODENAVIGATION && $NOCODENAVIGATION != "yes") {
    PY3_PROGRAM_LINK_EXE=$LINK_EXE && ${hide;kv:"py3yndex $TARGET"}
}
otherwise {
    PY3_PROGRAM_LINK_EXE=$LINK_EXE
}

# tag:python-specific
### @usage: _BASE_PY_PROGRAM #internal
###
### The base module for all Python 3.x binary programs. Adds linking logic, relevant module properties and
### dependency on Python 3.x interpreter. Also adds import tests on all sources including libraries.
### Links all Python 3.x libraries and Python 3.x interpreter into itself to form regular executable.
### This only compatible with PYTHON3-tagged modules and selects those from multimodules
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
module _BASE_PY3_PROGRAM: _BASE_PROGRAM {
    .CMD=$PY3_PROGRAM_LINK_EXE
    _ARCADIA_PYTHON3_ADDINCL()
    RUN_CYTHON_SCRIPT=$YMAKE_PYTHON3 $CYTHON_SCRIPT
    RUN_CYTHON_SCRIPT_H=$YMAKE_PYTHON3 $CYTHON_SCRIPT
    PYTHON_TYPE_FOR_CYTHON=PY3
    PEERDIR(library/python/runtime_py3/main)
    when ($PYTHON_SQLITE3 != "no") {
        PEERDIR += contrib/tools/python3/Modules/_sqlite
    }

    when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") {
        NO_CHECK_IMPORTS_FOR_VALUE=
    }
    NO_CLANG_TIDY()
    ADD_CHECK_PY_IMPORTS()

    when ($ARCH_PPC64LE == "yes") {
        _MY_ALLOCATOR=SYSTEM
    }
    otherwise {
        _MY_ALLOCATOR=J
    }
    ALLOCATOR($_MY_ALLOCATOR)
    STRIP()

    when ($NO_PYTHON_INCLS != "yes") {
        PEERDIR+=contrib/libs/python
    }
    when ($BUILD_TYPE == "DEBUG" || $BUILD_TYPE == "FASTDEBUG") {
        NO_STRIP=yes
    }
    when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") {
        NO_STRIP=yes
    }
    when ($PYTHON_COVERAGE == "yes") {
        PEERDIR+=library/python/coverage
    }
    when ($PYTHON_IMPORT_TRACING == "yes") {
        PEERDIR += library/python/import_tracing/constructor
    }
    when ($CODENAVIGATION && $NOCODENAVIGATION != "yes")  {
        PEERDIR += contrib/python/six
    }
    .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER
    .RESTRICTED=PYTHON3_ADDINCL USE_PYTHON3 PYTHON2_ADDINCL USE_PYTHON2
    SET(MODULE_LANG PY3)
    SET(ANTLR_PYTHON Python3)
}

# tag:python-specific
### @usage: PY3_PROGRAM_BIN([progname])
### Use instead of PY3_PROGRAM only if ya.make with PY3_PROGRAM() included in another ya.make
### In all other cases use PY3_PROGRAM
module PY3_PROGRAM_BIN: _BASE_PY3_PROGRAM {
    # Look's like we cannot avoid copy-paste util ymake supports multiple inheritance
    # We need to attach coverage.extractor to every py_program target, except pytest targets
    STYLE_FLAKE8()
    ADD_YTEST($MODULE_PREFIX$REALPRJNAME coverage.extractor)
}

# tag:python-specific tag:internal
NO_PYTHON_INCLS=no
### @usage: NO_PYTHON_INCLUDES() # internal
###
### Disable dependencies on libraries providing Python headers.
### This is only used in Python libraries themselves to avoid PEERDIR loops.
macro NO_PYTHON_INCLUDES() {
    ENABLE(NO_PYTHON_INCLS)
}

# tag:python-specific
### @usage: PYTHON2_ADDINCL()
###
### This macro adds include path for Python headers (Python 2.x variant) without PEERDIR.
### This should be used in 2 cases only:
### - In PY2MODULE since it compiles into .so and uses external Python runtime;
### - In system Python libraries themselves since proper PEERDIR there may create a loop;
### In all other cases use USE_PYTHON2 macro instead.
###
### Never use this macro in PY2_PROGRAM, PY2_LIBRARY and PY23_LIBRARY: they have everything needed by default.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs
macro PYTHON2_ADDINCL() {
    _PYTHON_ADDINCL()
    SET(MODULE_TAG PY2_NATIVE)
    SET(PEERDIR_TAGS CPP_PROTO CPP_PROTO_FROM_SCHEMA CPP_FBS CPP_ROS PY2_NATIVE YQL_UDF_STATIC __EMPTY__ RESOURCE_LIB DLL_LIB)
}

# tag:python-specific tag:internal
### @usage: _ARCADIA_PYTHON_ADDINCL()  # internal
### This macro sets up Python headers for modules with Arcadia python (e.g. PY2_LIBRARY) and configures module as Python 2.x.
macro _ARCADIA_PYTHON_ADDINCL() {
    _PYTHON_ADDINCL()
    SET(MODULE_TAG PY2)
    SET(PEERDIR_TAGS PY2 PY2_NATIVE PY_PROTO PY_PROTO_FROM_SCHEMA PY2_FBS YQL_UDF_STATIC __EMPTY__ RESOURCE_LIB DLL_LIB)
}

# tag:python-specific tag:internal
### @usage: _PYTHON_ADDINCL()  # internal
### This macro sets up Python 2.x headers for both Arcadia and non-Arcadia python.
macro _PYTHON_ADDINCL() {
    SET(PYTHON2 yes)
    SET(PYTHON3 no)
    SET(_PYTHON_VER py2)
    when ($USE_ARCADIA_PYTHON == "yes") {
        ADDINCL+=GLOBAL contrib/libs/python/Include
        CFLAGS+=-DARCADIA_PYTHON_UNICODE_SIZE=$ARCADIA_PYTHON_UNICODE_SIZE
    }
    otherwise {
        when ($USE_SYSTEM_PYTHON) {
            PEERDIR+=build/platform/python
        }
        otherwise {
            CFLAGS+=$PYTHON_INCLUDE
        }
    }
}

# tag:python-specific
### @usage: PYTHON3_ADDINCL()
###
### This macro adds include path for Python headers (Python 3.x variant).
### This should be used in 2 cases only:
### - In PY2MODULE since it compiles into .so and uses external Python runtime;
### - In system Python libraries themselves since peerdir there may create a loop;
### In all other cases use USE_PYTHON3() macro instead.
###
### Never use this macro in PY3_PROGRAM and PY3_LIBRARY and PY23_LIBRARY: they have everything by default.
###
### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs
macro PYTHON3_ADDINCL() {
    _PYTHON3_ADDINCL()
    SET(MODULE_TAG PY3_NATIVE)
    SET(PEERDIR_TAGS CPP_PROTO CPP_PROTO_FROM_SCHEMA CPP_FBS CPP_ROS PY3_NATIVE YQL_UDF_STATIC __EMPTY__ RESOURCE_LIB DLL_LIB)
}

# tag:python-specific tag:internal
### @usage: _ARCADIA_PYTHON3_ADDINCL()  # internal
###
### This macro sets up Python3 headers for modules with Arcadia python (e.g. PY3_LIBRARY) and configures module as Python 3.x.
macro _ARCADIA_PYTHON3_ADDINCL() {
    _PYTHON3_ADDINCL()
    SET(MODULE_TAG PY3)
    SET(PEERDIR_TAGS PY3 PY3_BIN_LIB PY3TEST_LIBRARY PY3_NATIVE PY3_PROTO PY3_PROTO_FROM_SCHEMA PY3_FBS PY3_ROS YQL_UDF_STATIC __EMPTY__ RESOURCE_LIB DLL_LIB)
}

# tag:python-specific  tag:internal
### @usage: _PYTHON_ADDINCL()  # internal
###
### This macro sets up Python 3.x headers for both Arcadia and non-Arcadia python.
macro _PYTHON3_ADDINCL() {
    SET(PYTHON3 yes)
    SET(PYTHON2 no)
    SET(_PYTHON_VER py3)
    when ($USE_ARCADIA_PYTHON == "yes") {
        CFLAGS+=-DUSE_PYTHON3
        ADDINCL+=GLOBAL contrib/libs/python/Include
    }
    otherwise {
        when ($USE_SYSTEM_PYTHON) {
            PEERDIR+=build/platform/python
        }
        otherwise {
            CFLAGS+=$PYTHON_INCLUDE
        }
    }
}

# tag:python-specific
### @usage: USE_PYTHON2()
###
### This adds Python 2.x runtime library to your LIBRARY and makes it Python2-compatible.
### Compatibility means proper PEERDIRs, ADDINCLs and variant selection on PEERDIRs to multimodules.
###
### If you'd like to use #include <Python.h> with Python2 specify USE_PYTHON2 or better make it PY2_LIBRARY.
### If you'd like to use #include <Python.h> with Python3 specify USE_PYTHON3 or better make it PY3_LIBRARY.
### If you'd like to use #include <Python.h> with both Python2 and Python3 convert your LIBRARY to PY23_LIBRARY.
###
### @see: [PY2_LIBRARY](#module_PY2_LIBRARY), [PY3_LIBRARY](#module_PY3_LIBRARY), [PY23_LIBRARY](#multimodule_PY23_LIBRARY)
macro USE_PYTHON2() {
    _ARCADIA_PYTHON_ADDINCL()
    SET(PEERDIR_TAGS PY2 PY2_NATIVE CPP_PROTO CPP_PROTO_FROM_SCHEMA CPP_FBS CPP_ROS YQL_UDF_STATIC __EMPTY__ RESOURCE_LIB DLL_LIB)
    PEERDIR(contrib/libs/python)
}

# tag:python-specific
### @usage: USE_PYTHON3()
###
### This adds Python3 library to your LIBRARY and makes it Python3-compatible.
### Compatibility means proper PEERDIRs, ADDINCLs and variant selection on PEERDIRs to multimodules.
###
### If you'd like to use #include <Python.h> with Python3 specify USE_PYTHON3 or better make it PY3_LIBRARY.
### If you'd like to use #include <Python.h> with Python2 specify USE_PYTHON2 or better make it PY2_LIBRARY.
### If you'd like to use #include <Python.h> with both Python2 and Python3 convert your LIBRARY to PY23_LIBRARY.
###
### @see: [PY2_LIBRARY](#module_PY2_LIBRARY), [PY3_LIBRARY](#module_PY3_LIBRARY), [PY23_LIBRARY](#multimodule_PY23_LIBRARY)
macro USE_PYTHON3() {
    _ARCADIA_PYTHON3_ADDINCL()
    SET(PEERDIR_TAGS PY3 PY3_BIN_LIB PY3TEST_LIBRARY PY3_NATIVE CPP_PROTO CPP_PROTO_FROM_SCHEMA CPP_FBS CPP_ROS YQL_UDF_STATIC __EMPTY__ RESOURCE_LIB DLL_LIB)
    PEERDIR(contrib/libs/python)

    when ($USE_ARCADIA_PYTHON == "yes") {
        PEERDIR+=library/python/runtime_py3
    }
}

# tag:python-specific tag:coverage
### @usage: NO_PYTHON_COVERAGE()
###
### Disable python coverage for module
macro NO_PYTHON_COVERAGE() {
    DISABLE(PYTHON_COVERAGE)
}

# tag:python-specific tag:import_tracing
### @usage: NO_IMPORT_TRACING()
###
### Disable python coverage for module
macro NO_IMPORT_TRACING() {
    DISABLE(PYTHON_IMPORT_TRACING)
}

# tag:python-specific tag:coverage tag:cython
### @usage: NO_CYTHON_COVERAGE()
###
### Disable cython and cythonized python coverage (CYTHONIZE_PY)
### Implies NO_CLANG_COVERAGE() - right now, we can't disable instrumentation for .py.cpp files, but enable for .cpp
macro NO_CYTHON_COVERAGE() {
    DISABLE(CYTHON_COVERAGE)
    NO_CLANG_COVERAGE()
}

# tag:python-specific
### @usage: PY_SRCS({| CYTHON_C} { | TOP_LEVEL | NAMESPACE ns} Files...)
###
### Build specified Python sources according to Arcadia binary Python build. Basically creates precompiled and source resources keyed with module paths.
### The resources eventually are linked into final program and can be accessed as regular Python modules.
### This custom loader linked into the program will add them to sys.meta_path.
###
### PY_SRCS also support .proto, .ev, .pyx and .swg files. The .proto and .ev are compiled to .py-code by protoc and than handled as usual .py files.
### .pyx and .swg lead to C/C++ Python extensions generation, that are automatically registered in Python as built-in modules.
###
### By default .pyx files are built as C++-extensions. Use CYTHON_C to build them as C (similar to BUILDWITH_CYTHON_C, but with the ability to specify namespace).
###
### __init__.py never required, but if present (and specified in PY_SRCS), it will be imported when you import package modules with __init__.py Oh.
###
### @example
###
###     PY2_LIBRARY(mymodule)
###         PY_SRCS(a.py sub/dir/b.py e.proto sub/dir/f.proto c.pyx sub/dir/d.pyx g.swg sub/dir/h.swg)
###     END()
###
### PY_SRCS honors Python2 and Python3 differences and adjusts itself to Python version of a current module.
### PY_SRCS can be used in any Arcadia Python build modules like PY*_LIBRARY, PY*_PROGRAM, PY*TEST.
### PY_SRCS in LIBRARY or PROGRAM effectively converts these into PY2_LIBRARY and PY2_PROGRAM respectively.
### It is strongly advised to make this conversion explicit. Never use PY_SRCS in a LIBRARY if you plan to use it from external Python extension module.
###
### Documentation: https://wiki.yandex-team.ru/arcadia/python/pysrcs/#modulipylibrarypy3libraryimakrospysrcs
macro PY_SRCS() {
    DEFAULT(MODULE_TAG PY2)
}

# tag:python-specific
_ALL_PY_FILES=

# tag:python-specific
macro _ALL_PY_SRCS2(TOP_LEVEL?"TOP_LEVEL":"", RECURSIVE?"/**":"", ONLY_TEST_FILES?"test_*.py":"*.py", ONLY_TEST_FILES2?"*_test.py":"***", NO_TEST_FILES?"**/test_*.py **/*_test.py":"", NAMESPACE[], REST[], REST2[], EAT_TAIL[]) {
    _GLOB(_ALL_PY_FILES ${suf=${RECURSIVE}/${ONLY_TEST_FILES}:REST} ${suf=${RECURSIVE}/${ONLY_TEST_FILES2}:REST2} ${suf=${RECURSIVE}/ya.make:REST} EXCLUDE ya.make ${NO_TEST_FILES})
}

# tag:python-specific
### @usage: ALL_PY_SRCS([RECURSIVE] [NO_TEST_FILES] { | TOP_LEVEL | NAMESPACE ns} [Dirs...])
###
### Puts all .py-files from given Dirs (relative to projects') into PY_SRCS of the current module.
### If Dirs is ommitted project directory is used
###
### `RECURSIVE` makes lookup recursive with resprect to Dirs
### `NO_TEST_FILES` excludes files `test_*.py` and `*_test.py` those are normally subject to `TEST_SRCS`
### `TOP_LEVEL` and `NAMESPACE` are forwarded to `PY_SRCS`
###
### Note: Only one such macro per module is allowed
### Note: Macro is designed to reject any ya.make files in Dirs except current one
###
### @see [PY_SRCS()](#macro_PY_SRCS)
macro ALL_PY_SRCS(TOP_LEVEL?"TOP_LEVEL":"", NAMESPACE="", RECURSIVE?"RECURSIVE":"", NO_TEST_FILES?"NO_TEST_FILES":"", Files...) {
    _ALL_PY_SRCS2(${pre=NAMESPACE :NAMESPACE} ${TOP_LEVEL} ${RECURSIVE} ${NO_TEST_FILES} REST .${pre=/:Files})
    PY_SRCS(_MR ${pre=NAMESPACE :NAMESPACE} ${TOP_LEVEL} $_ALL_PY_FILES)
}

# tag:python-specific
### @usage: ALL_PYTEST_SRCS([RECURSIVE] [Dirs...])
###
### Puts all .py-files from given Dirs (relative to projects') into TEST_SRCS of the current module.
### If Dirs is omitted project directory is used
###
### `RECURSIVE` makes lookup recursive with respect to Dirs
### `ONLY_TEST_FILES` includes only files `test_*.py` and `*_test.py`, others are normally subject to `PY_SRCS`
###
### Note: Only one such macro per module is allowed
### Note: Macro is designed to reject any ya.make files in Dirs except current one
###
### @see [TEST_SRCS()](#macro_TEST_SRCS)
macro ALL_PYTEST_SRCS(TOP_LEVEL?"TOP_LEVEL":"", NAMESPACE="", RECURSIVE?"RECURSIVE":"", ONLY_TEST_FILES?"ONLY_TEST_FILES ONLY_TEST_FILES2 REST2":"EAT_TAIL", Files...) {
    _ALL_PY_SRCS2(${pre=NAMESPACE :NAMESPACE} ${TOP_LEVEL} ${RECURSIVE} REST .${pre=/:Files} ${ONLY_TEST_FILES} .${pre=/:Files})
    TEST_SRCS(_MR $_ALL_PY_FILES)
}

# tag:python-specific
_PY_EXTRA_LINT_FILES_VALUE=
### @usage: PY_EXTRA_LINT_FILES(files...)
###
### Add extra Python files for linting. This macro allows adding
### Python files which has no .py extension.
macro PY_EXTRA_LINT_FILES(FILES...) {
    SET_APPEND(_PY_EXTRA_LINT_FILES_VALUE $FILES)
}

# tag:python-specific
### @usage: PY23_LIBRARY([name])
###
### Build PY2_LIBRARY or PY3_LIBRARY depending on incoming PEERDIR.
### Direct build or build by RECURSE creates both variants.
### This multimodule doesn't define any final targets, so use from DEPENDS or BUNDLE is not allowed.
###
### Documentation: https://wiki.yandex-team.ru/arcadia/python/pysrcs
multimodule PY23_LIBRARY {
    module PY2: PY2_LIBRARY {
        .RESTRICTED=RUN_ANTLR4_PYTHON
        OBJ_SUF=.py2
    }
    module PY3: PY3_LIBRARY {
        .RESTRICTED=RUN_ANTLR4_PYTHON
        RUN_CYTHON_SCRIPT_H=$YMAKE_PYTHON $CYTHON_SCRIPT
        OBJ_SUF=.py3
    }
}

# tag:python-specific
### @usage: PY23_NATIVE_LIBRARY([name])
###
### Build LIBRARY compatible with either Python 2.x or Python 3.x depending on incoming PEERDIR.
###
### This multimodule doesn't depend on Arcadia Python binary build. It is intended only for C++ code and cannot contain PY_SRCS and USE_PYTHON2 macros.
### Use these multimodule instead of PY23_LIBRARY if the C++ extension defined in it will be used in PY2MODULE.
### While it doesn't bring Arcadia Python dependency itself, it is still compatible with Arcadia Python build and can be PEERDIR-ed from PY2_LIBRARY and alikes.
### Proper version will be selected according to Python version of the module PEERDIR comes from.
###
### This mulrtimodule doesn't define any final targets so cannot be used from DEPENDS or BUNDLE macros.
###
### For more information read https://wiki.yandex-team.ru/arcadia/python/pysrcs/#pysrcssrcsipy23nativelibrary
###
### @see [LIBRARY()](#module_LIBRARY), [PY2MODULE()](#module_PY2MODULE)
multimodule PY23_NATIVE_LIBRARY {
    module PY2: LIBRARY {
        .RESTRICTED=PY_SRCS USE_PYTHON2 USE_PYTHON3 PYTHON3_ADDINCL RUN_ANTLR4_PYTHON
        OBJ_SUF=.py2
        PYTHON2_ADDINCL()
        SET(MODULE_LANG PY2)
    }
    module PY3: LIBRARY {
        .RESTRICTED=PY_SRCS USE_PYTHON2 USE_PYTHON3 RUN_ANTLR4_PYTHON
        .ALIASES=PYTHON2_ADDINCL=PYTHON3_ADDINCL
        .SEM=CPP_LIBRARY_SEM
        .GLOBAL_SEM=CPP_OBJ_LIBRARY_SEM
        PYTHON3_ADDINCL()
        SET(MODULE_LANG PY3)
        when ($MSVC == "yes" || $CYGWIN == "yes") {
            MODULE_PREFIX=py3c
        }
        otherwise {
            MODULE_PREFIX=libpy3c
        }
        OBJ_SUF=.py3
    }
}

# tag:python-specific
multimodule PY23_TEST {
    module PY2 : PYTEST_BIN {
        .RESTRICTED=RUN_ANTLR4_PYTHON
        MODULE_PREFIX=py2_
        OBJ_SUF=.py2
        CANONIZE_SUB_PATH=py2test
        RUN_CYTHON_SCRIPT_H=$YMAKE_PYTHON $CYTHON_SCRIPT

        SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS canondata/py2test/result.json)

        _DONT_REQUIRE_LICENSE()
        WITHOUT_VERSION()
    }
    module PY3TEST_PROGRAM: PY3TEST_BIN {
        .FINAL_TARGET=yes
        .RESTRICTED=RUN_ANTLR4_PYTHON
        OBJ_SUF=.py3
        CANONIZE_SUB_PATH=py3test
        RUN_CYTHON_SCRIPT_H=$YMAKE_PYTHON $CYTHON_SCRIPT

        SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS canondata/py3test/result.json)

        _DONT_REQUIRE_LICENSE()
        WITHOUT_VERSION()
    }

    module PY3TEST_LIBRARY: PY3_LIBRARY {
        .RESTRICTED=RUN_ANTLR4_PYTHON
        PEERDIR+=library/python/pytest
        RUN_CYTHON_SCRIPT_H=$YMAKE_PYTHON $CYTHON_SCRIPT

        _DONT_REQUIRE_LICENSE()
        WITHOUT_VERSION()
    }
}