diff options
author | snermolaev <snermolaev@yandex-team.com> | 2023-08-06 17:52:03 +0300 |
---|---|---|
committer | snermolaev <snermolaev@yandex-team.com> | 2023-08-06 18:59:54 +0300 |
commit | a38bc05b605db06f9569320966726795d2f6476e (patch) | |
tree | d542b58c8046c4a5a1f0895c272c5e0e3f9912ea /build | |
parent | d1257d50ffa27ba0ba788a7dd11a797bfbdad0c5 (diff) | |
download | ydb-a38bc05b605db06f9569320966726795d2f6476e.tar.gz |
add proto.conf for PROTO
Diffstat (limited to 'build')
-rw-r--r-- | build/conf/proto.conf | 774 | ||||
-rw-r--r-- | build/ymake.core.conf | 781 |
2 files changed, 778 insertions, 777 deletions
diff --git a/build/conf/proto.conf b/build/conf/proto.conf new file mode 100644 index 0000000000..cf97fda7a1 --- /dev/null +++ b/build/conf/proto.conf @@ -0,0 +1,774 @@ +# tag:tool-specific tag:proto +PROTOC=${tool:"contrib/tools/protoc"} +PROTOC_STYLEGUIDE_OUT=--cpp_styleguide_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE +PROTOC_PLUGIN_STYLEGUIDE=--plugin=protoc-gen-cpp_styleguide=${tool:"contrib/tools/protoc/plugins/cpp_styleguide"} +PROTOBUF_PATH=${ARCADIA_ROOT}/contrib/libs/protobuf/src +USE_VANILLA_PROTOC=no +PROTOC_TRANSITIVE_HEADERS=yes +_PROTOC_FLAGS= + +# tag:proto +PY_PROTOS_FOR=no +BUILD_PROTO_AS_EVLOG=no + +# tag:proto +PROTO_NAMESPACE= + +# tag:proto tag:python-specific +GEN_PY_PROTOS=$YMAKE_PYTHON ${input:"build/scripts/gen_py_protos.py"} + +# tag:proto tag:cpp-specific +PROTO_HEADER_EXTS=.pb.h +CPP_PROTO_CMDLINE= +CPP_PROTO_OPTS= +CPP_PROTO_OUTS= +CPP_PROTO_OUTS_SEM= +CPP_PROTO_SUFFIXES=.pb.h .pb.cc +CPP_PROTO_PLUGINS= + +# tag:proto tag:cpp-specific +CPP_EV_OPTS=--plugin=protoc-gen-event2cpp=${tool:"tools/event2cpp"} --event2cpp_out=$ARCADIA_BUILD_ROOT -I=$ARCADIA_ROOT/library/cpp/eventlog +CPP_EV_OUTS= +CPP_EV_OUTS_SEM= + +# tag:proto tag:python-specific +PY_PROTO_OPTS= +PY_PROTO_OUTS= +PY_PROTO_OUTS_INTERNAL= +PY_PROTO_DEPS= +PY_PROTO_SUFFIXES=_pb2.py +PY_EVLOG_SUFFIXES=_ev_pb2.py + +# tag:proto tag:java-specific +JAVA_PROTO_ARGS= + +# tag:proto tag:python-specific +OPTIMIZE_PY_PROTOS_FLAG=no + +# tag:internal +### @usage: _ORDER_ADDINCL([BUILD ...] [SOURCE ...] Args...) # internal +### +### Order and filter ADDINCLs (Args - is intentionally omitted in ADDINCL macro) +macro _ORDER_ADDINCL(BUILD[], SOURCE[], ARGS...) { + ADDINCL($BUILD $SOURCE) +} + +# tag:proto +### @usage: PROTO_ADDINCL([GLOBAL] [WITH_GEN] Path) +### +### This macro introduces proper ADDINCLs for .proto-files found in sources and +### .cpp/.h generated files, supplying them to appropriate commands and allowing +### proper dependency resolution at configure-time. +### +### Note: you normally shouldn't use this macro. ADDINCLs should be sent to user +### from dependency via PROTO_NAMESPACE macro +macro PROTO_ADDINCL(GLOBAL?"GLOBAL":"", Path, WITH_GEN?"BUILD":"") { + _ORDER_ADDINCL($WITH_GEN $GLOBAL FOR proto ${ARCADIA_BUILD_ROOT}/$Path SOURCE $GLOBAL FOR proto ${ARCADIA_ROOT}/$Path) + ADDINCL($GLOBAL ${ARCADIA_BUILD_ROOT}/$Path) +} + +# tag:proto +### @usage: PROTO_NAMESPACE([WITH_GEN] Namespace) +### +### Defines protobuf namespace (import/export path prefix) which should be used for imports and +### which defines output path for .proto generation. +### +### For proper importing and configure-time dependency management it sets ADDINCLs +### for both .cpp headers includes and .proto imports. If .proto expected to be used outside of the +### processing module use GLOBAL to send proper ADDINCLs to all (transitive) users. PEERDIR to +### PROTO_LIBRARY with PROTO_NAMESPACE() is enough at user side to correctly use the library. +### If generated .proto files are going to be used for building a module than use of WITH_GEN +### parameter will add appropriate dir from the build root for .proto files search. +macro PROTO_NAMESPACE(GLOBAL?"GLOBAL":"", WITH_GEN?"WITH_GEN":"", Namespace) { + SET(PROTO_NAMESPACE $Namespace) + PROTO_ADDINCL(GLOBAL $WITH_GEN $Namespace) +} + +# tag:proto +### @usage PROTOC_FATAL_WARNINGS() +### +### Treat protoc warnings as fatal errors that break the build, for example, unused imports +### Adds `--fatal_warnings` argument to protoc +macro PROTOC_FATAL_WARNINGS() { + SET_APPEND(_PROTOC_FLAGS --fatal_warnings) +} + +# tag:proto tag:internal tag:python-specific +### @usage: OPTIMIZE_PY_PROTOS() # internal +### +### Enable Python proto optimization by embedding corresponding C++ code into binary. +### Python protobuf runtime will use C++ implementation instead of Python one if former is available. +### This is default mode for most PROTO_LIBRARY's and PY2_LIBRARY's, some system ones being exceptions. +macro OPTIMIZE_PY_PROTOS() { + SET(OPTIMIZE_PY_PROTOS_FLAG yes) +} + +# tag:proto tag:python-specific +### @usage: NO_OPTIMIZE_PY_PROTOS() +### +### Disable Python proto optimization using embedding corresponding C++ code into binary. +### Python protobuf runtime will use C++ implementation instead of Python one if former is available. +### This is default mode only for some system libraries. +macro NO_OPTIMIZE_PY_PROTOS() { + SET(OPTIMIZE_PY_PROTOS_FLAG no) +} + +# tag:proto tag:python-specific +macro _PROTO_PLUGIN_ARGS_BASE(Name, Tool, OutParm...) { + .CMD=--plugin=protoc-gen-${Name}=\${tool:"$Tool"} --${Name}_out=$ARCADIA_BUILD_ROOT/\$PROTO_NAMESPACE ${pre=--${Name}_opt=:OutParm} + .SEM=IGNORED +} + +# tag:proto tag:python-specific +macro _ADD_PY_PROTO_OUT(Suf) { + SET_APPEND(PY_PROTO_OUTS \${output;hide;noauto;norel;nopath;noext;suf=$Suf:File}) + SET_APPEND(PY_PROTO_OUTS_INTERNAL \${output;hide;noauto;norel;nopath;noext;suf=__int__$Suf:File} \${hide;kv:"ext_out_name_for_\${nopath;noext;suf=__int__$Suf:File} \${nopath;noext;suf=$Suf:File}"}) + # XXX fix variable expansion in plugins + SET(PY_PROTO_SUFFIXES $PY_PROTO_SUFFIXES $Suf) +} + +# tag:proto tag:python-specific +### @usage: PY_PROTO_PLUGIN(Name Ext Tool DEPS <Dependencies>) +### +### Define protoc plugin for python with given Name that emits extra output with provided Extension +### using Tool. Extra dependencies are passed via DEPS +macro PY_PROTO_PLUGIN(NAME, EXT, TOOL, DEPS[]) { + SET_APPEND(PY_PROTO_OPTS $_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL)) + _ADD_PY_PROTO_OUT($EXT) + # XXX fix variable expansion in plugins + SET(PY_PROTO_DEPS $PY_PROTO_DEPS $DEPS) +} + +# tag:proto tag:python-specific +### @usage: PY_PROTO_PLUGIN2(Name Ext1 Ext2 Tool DEPS <Dependencies>) +### +### Define protoc plugin for python with given Name that emits 2 extra outputs with provided Extensions +### using Tool. Extra dependencies are passed via DEPS +macro PY_PROTO_PLUGIN2(NAME, EXT1, EXT2, TOOL, DEPS[]) { + PY_PROTO_PLUGIN($NAME $EXT1 $TOOL DEPS $DEPS) + _ADD_PY_PROTO_OUT($EXT2) +} + +# tag:proto tag:java-specific +macro _JAVA_PROTO_PLUGIN_ARGS_BASE(Name, Tool, OutParm...) { + .CMD=--plugin=protoc-gen-${Name}=\${tool:"$Tool"} --${Name}_out=$ARCADIA_BUILD_ROOT/java_out +} + +# tag:proto tag:java-specific +### @usage: JAVA_PROTO_PLUGIN(Name Tool DEPS <Dependencies>) +### +### Define protoc plugin for Java with given Name that emits extra outputs +### using Tool. Extra dependencies are passed via DEPS +macro JAVA_PROTO_PLUGIN(NAME, TOOL, DEPS[]) { + SET_APPEND(JAVA_PROTO_ARGS $_JAVA_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL)) + # XXX fix variable expansion in plugins + SET(JAVA_PROTOBUF_PEERS $JAVA_PROTOBUF_PEERS $DEPS) +} + +macro _ADD_SEM_PROP_IF_NON_EMPTY(Prop, Args...) { + .SEM=${pre=&& set_target_property $Prop :Args} +} + +# tag:proto tag:cpp-specific +macro _ADD_CPP_PROTO_OUT(Suf) { + .SEM=append_target_property PROTOC_EXTRA_OUTS $Suf ${output;hide;suf=.o:Suf} $_ADD_SEM_PROP_IF_NON_EMPTY(PROTO_NAMESPACE $PROTO_NAMESPACE) + SET_APPEND(CPP_PROTO_OUTS \${output;hide;norel;nopath;noext;suf=$Suf:File}) + + # XXX fix variable expansion in plugins + SET(CPP_PROTO_SUFFIXES $CPP_PROTO_SUFFIXES $Suf) +} + +# tag:proto tag:cpp-specific +HAS_CPP_PROTOBUF_PEERS=no +CPP_PROTOBUF_PEERS= + +# tag:proto tag:cpp-specific +### @usage: CPP_PROTO_PLUGIN0(Name Tool DEPS <Dependencies>) +### +### Define protoc plugin for C++ with given Name that emits code into regular outputs +### using Tool. Extra dependencies are passed via DEPS. +macro CPP_PROTO_PLUGIN0(NAME, TOOL, DEPS[]) { + .SEM=target_proto_plugin $NAME ${tool;rootrel:TOOL} ${output;hide;suf=.fake.o:NAME} + SET_APPEND(CPP_PROTO_OPTS $_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL)) + + # XXX fix variable expansion in plugins + ENABLE(HAS_CPP_PROTOBUF_PEERS) + SET(CPP_PROTOBUF_PEERS $CPP_PROTOBUF_PEERS $DEPS) +} + +# tag:proto tag:cpp-specific +### @usage: CPP_PROTO_PLUGIN(Name Tool Suf DEPS <Dependencies>) +### +### Define protoc plugin for C++ with given Name that emits code into 1 extra output +### using Tool. Extra dependencies are passed via DEPS. +macro CPP_PROTO_PLUGIN(NAME, TOOL, SUF, DEPS[]) { + CPP_PROTO_PLUGIN0($NAME $TOOL DEPS $DEPS) + + _ADD_CPP_PROTO_OUT($SUF) +} + +# tag:proto tag:cpp-specific +### @usage: CPP_PROTO_PLUGIN2(Name Tool Suf1 Suf2 DEPS <Dependencies>) +### +### Define protoc plugin for C++ with given Name that emits code into 2 extra outputs +### using Tool. Extra dependencies are passed via DEPS. +macro CPP_PROTO_PLUGIN2(NAME, TOOL, SUF1, SUF2, DEPS[]) { + CPP_PROTO_PLUGIN($NAME $TOOL $SUF1 DEPS $DEPS) + + _ADD_CPP_PROTO_OUT($SUF2) + SET_APPEND(CPP_PROTO_OUTS_SEM \${output;hide;norel;nopath;noext;suf=$SUF2:File}) +} + +# tag:proto +### @usage: USE_SKIFF() #wip, do not use +### +### Use mapreduce/yt/skiff_proto/plugin for C++ +macro USE_SKIFF() { + # Move extra includes to INDUCED_DEPS macro in mapreduce/yt/skiff_proto/plugin/ya.make + CPP_PROTO_PLUGIN0(skiff mapreduce/yt/skiff_proto/plugin DEPS mapreduce/yt/skiff_proto/lib) +} + +# tag:go-specific +_GO_COMMON_GOOGLE_APIS = \ +vendor/google.golang.org/genproto/googleapis/api/annotations \ +vendor/google.golang.org/genproto/googleapis/api/configchange \ +vendor/google.golang.org/genproto/googleapis/api/distribution \ +vendor/google.golang.org/genproto/googleapis/api/expr/v1alpha1 \ +vendor/google.golang.org/genproto/googleapis/api/expr/v1beta1 \ +vendor/google.golang.org/genproto/googleapis/api/httpbody \ +vendor/google.golang.org/genproto/googleapis/api/label \ +vendor/google.golang.org/genproto/googleapis/api/metric \ +vendor/google.golang.org/genproto/googleapis/api/monitoredres \ +vendor/google.golang.org/genproto/googleapis/api/serviceconfig \ +vendor/google.golang.org/genproto/googleapis/api/servicecontrol/v1 \ +vendor/google.golang.org/genproto/googleapis/api/servicemanagement/v1 \ +vendor/google.golang.org/genproto/googleapis/iam/admin/v1 \ +vendor/google.golang.org/genproto/googleapis/iam/credentials/v1 \ +vendor/google.golang.org/genproto/googleapis/iam/v1 \ +vendor/google.golang.org/genproto/googleapis/iam/v1/logging \ +vendor/google.golang.org/genproto/googleapis/logging/type \ +vendor/google.golang.org/genproto/googleapis/logging/v2 \ +vendor/google.golang.org/genproto/googleapis/rpc/code \ +vendor/google.golang.org/genproto/googleapis/rpc/errdetails \ +vendor/google.golang.org/genproto/googleapis/rpc/status \ +vendor/google.golang.org/genproto/googleapis/type/calendarperiod \ +vendor/google.golang.org/genproto/googleapis/type/color \ +vendor/google.golang.org/genproto/googleapis/type/date \ +vendor/google.golang.org/genproto/googleapis/type/dayofweek \ +vendor/google.golang.org/genproto/googleapis/type/expr \ +vendor/google.golang.org/genproto/googleapis/type/fraction \ +vendor/google.golang.org/genproto/googleapis/type/latlng \ +vendor/google.golang.org/genproto/googleapis/type/money \ +vendor/google.golang.org/genproto/googleapis/type/postaladdress \ +vendor/google.golang.org/genproto/googleapis/type/quaternion \ +vendor/google.golang.org/genproto/googleapis/type/timeofday + +# tag:go-specific +_COMMON_GOOGLE_APIS=None + +# tag:go-specific +### @usage: USE_COMMON_GOOGLE_APIS([apis...]) +### +### Use common-google-apis library set. Pass optional apis list to be used or use them all. +### This macro is properly handled for all languages including Go, where apis come in +## pregenerated form. In other languages apis are generated from sources in Arcadia. +macro USE_COMMON_GOOGLE_APIS(APIS...) { + SET(_COMMON_GOOGLE_APIS ${pre=vendor/google.golang.org/genproto/googleapis/:APIS}) +} + +# tag:go-specific tag:proto +GO_PROTO_GEN_PLUGINS= +GO_PROTO_OPTS= +GO_PROTO_OUTS= +GO_PROTO_GRPC_OPTS= +GO_PROTO_GRPC_OUTS= +GO_PROTO_V2=no +_GO_PROTO_CHECK_OUTPUT= + +# tag:go-specific tag:proto +GO_PROTOBUF_IMPORTS=\ + ${GOSTD}/sync \ + ${GOSTD}/reflect \ + ${GOSTD}/fmt \ + ${GOSTD}/math \ + vendor/github.com/golang/protobuf/proto \ + vendor/google.golang.org/protobuf/types/descriptorpb \ + vendor/google.golang.org/protobuf/runtime/protoimpl \ + vendor/google.golang.org/protobuf/runtime/protoiface \ + vendor/google.golang.org/protobuf/reflect/protoreflect + +# tag:go-specific tag:proto +GO_PROTOBUF_WELLKNOWN_TYPES=\ + vendor/google.golang.org/protobuf/types/known/anypb \ + vendor/google.golang.org/protobuf/types/known/apipb \ + vendor/google.golang.org/protobuf/types/known/durationpb \ + vendor/google.golang.org/protobuf/types/known/emptypb \ + vendor/google.golang.org/protobuf/types/known/fieldmaskpb \ + vendor/google.golang.org/protobuf/types/known/sourcecontextpb \ + vendor/google.golang.org/protobuf/types/known/structpb \ + vendor/google.golang.org/protobuf/types/known/timestamppb \ + vendor/google.golang.org/protobuf/types/known/typepb \ + vendor/google.golang.org/protobuf/types/known/wrapperspb + +# tag:go-specific tag:proto +### @usage: GO_PROTO_PLUGIN(Name Ext Tool [DEPS dependencies...]) +### +### Define protoc plugin for GO with given Name that emits extra output with provided extension +### Ext using Tool. Extra dependencies are passed via DEPS. +macro GO_PROTO_PLUGIN(NAME, EXT, TOOL, DEPS[]) { + SET_APPEND(GO_PROTO_OPTS $_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL)) + SET_APPEND(GO_PROTO_OUTS \${output;hide;noauto;norel;nopath;noext;suf=$EXT:File}) + PEERDIR(${DEPS}) +} + +# tag:go-specific tag:proto +GO_PROTO_CMDLINE=${cwd;rootdir;input:File} $YMAKE_PYTHON3 ${input:"build/scripts/go_proto_wrapper.py"} --arcadia-prefix $GO_ARCADIA_PROJECT_PREFIX --contrib-prefix $GO_CONTRIB_PROJECT_PREFIX --namespace ./$PROTO_NAMESPACE $_GO_PROTO_CHECK_OUTPUT --proto ${input;rootrel:File} -- $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_PATH $_PROTOC_FLAGS + +# tag:go-specific tag:proto +macro _GO_PROTO_CMD_IMPL(File, OPTS...) { + .CMD=$GO_PROTO_CMDLINE $OPTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} + .PEERDIR=${GO_PROTOBUF_IMPORTS} ${GO_PROTOBUF_WELLKNOWN_TYPES} + .ADDINCL=FOR proto contrib/libs/protobuf/src +} + +# tag:go-specific tag:proto +macro _GO_PROTO_CMD(File) { + .CMD=$_GO_PROTO_CMD_IMPL($File $GO_PROTO_OPTS $GO_PROTO_OUTS) +} + +# tag:proto +### @usage: YP_PROTO_YSON(Files... OUT_OPTS Opts...) +### +### Generate .yson.go from .proto using yp/go/yson/internal/proto-yson-gen/cmd/proto-yson-gen +macro YP_PROTO_YSON(OUT_OPTS[], Files...) { + .CMD=${cwd:BINDIR} $PROTOC --plugin=protoc-gen-custom=${tool:"yp/go/yson/internal/proto-yson-gen/cmd/proto-yson-gen"} -I=${ARCADIA_ROOT}/${PROTO_NAMESPACE} ${pre=-I=:_PROTO__INCLUDE} -I=${ARCADIA_ROOT} --custom_out="$OUT_OPTS paths=base_name:." --custom_opt="goroot=${GO_TOOLS_ROOT}" $_PROTOC_FLAGS ${input:Files} ${output;hide;noauto;nopath;noext;suf=.yson.go:Files} + .ADDINCL=FOR proto ${ARCADIA_ROOT}/${MODDIR} FOR proto ${ARCADIA_ROOT}/${GO_TEST_IMPORT_PATH} FOR proto yt ${ARCADIA_BUILD_ROOT}/yt FOR proto contrib/libs/protobuf/src + .PEERDIR=$GOSTD/strings $GOSTD/fmt $GOSTD/errors $GOSTD/encoding/json library/go/core/xerrors yt/go/yson yt/go/yterrors yp/go/yson/ytypes contrib/libs/protobuf + +} + +_SEM_CPP_PROTO_CMD=target_proto_messages PRIVATE ${input:File} $CPP_PROTO_OUTS_SEM ${output;hide;suf=${OBJ_SUF}.pb.o:File} ${hide;tool:"contrib/tools/protoc/bin"} ${hide;tool:"contrib/tools/protoc/plugins/cpp_styleguide"} \ + && set_global_flags COMMON_PROTOC_FLAGS \ + && target_proto_outs --cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $PROTOC_STYLEGUIDE_OUT \ + && target_proto_addincls ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE ${_PROTO__INCLUDE} $ARCADIA_BUILD_ROOT $PROTOBUF_PATH + +# tag:proto +macro _CPP_PROTO_CMD(File) { + .CMD=$CPP_PROTO_CMDLINE $CPP_PROTO_OPTS $CPP_PROTO_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} + .SEM=$_SEM_CPP_PROTO_CMD + .PEERDIR=contrib/libs/protobuf +} + +# tag:proto +macro _CPP_VANILLA_PROTO_CMD(File) { + .CMD=$CPP_PROTO_CMDLINE $CPP_PROTO_OPTS $CPP_PROTO_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} + .PEERDIR=contrib/libs/protobuf_std +} + +_SEM_CPP_EV_CMD=target_ev_messages PRIVATE ${input:File} $CPP_EV_OUTS_SEM ${output;hide;suf=${OBJ_SUF}.pb.o:File} ${hide;tool:"tools/event2cpp/bin"} ${hide;tool:"contrib/tools/protoc/bin"} ${hide;tool:"contrib/tools/protoc/plugins/cpp_styleguide"} \ + && set_global_flags COMMON_PROTOC_FLAGS \ + && target_proto_outs --cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $PROTOC_STYLEGUIDE_OUT --event2cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE \ + && target_proto_addincls ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE ${_PROTO__INCLUDE} $ARCADIA_BUILD_ROOT $PROTOBUF_PATH $ARCADIA_ROOT/library/cpp/eventlog + +# tag:proto +macro _CPP_EVLOG_CMD(File) { + .CMD=$CPP_PROTO_CMDLINE $CPP_EV_OPTS $CPP_EV_OUTS ${kv;hide:"p EV"} ${kv;hide:"pc yellow"} + .SEM=$_SEM_CPP_EV_CMD + .PEERDIR=library/cpp/eventlog contrib/libs/protobuf +} + +# tag:proto +macro _CPP_PROTO_EVLOG_CMD(File) { + # process .proto as .ev + .CMD=$CPP_PROTO_CMDLINE $CPP_EV_OPTS $CPP_PROTO_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} + .PEERDIR=library/cpp/eventlog contrib/libs/protobuf +} + +# tag:proto +macro _CPP_CFGPROTO_CMD(File) { + # keep extension in output just as in EV: this is hard-coded behaviour of protoc for non-.proto extensions + .CMD=$CPP_PROTO_CMDLINE --plugin=protoc-gen-config=${tool:"library/cpp/proto_config/plugin"} --config_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $CPP_EV_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} + .PEERDIR=library/cpp/proto_config/codegen library/cpp/proto_config/protos contrib/libs/protobuf +} + +# tag:proto +PY_PROTO_MYPY_ENABLED=yes +PY_PROTO_MYPY_PLUGIN_BASE=--plugin=protoc-gen-mypy=${tool:"contrib/python/mypy-protobuf/bin/protoc-gen-mypy"} --mypy_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE + +# tag:proto tag:python-specific +PY_PROTO_MYPY_SUFFIX= +PY_PROTO_MYPY_PLUGIN= +PY_PROTO_MYPY_PLUGIN_INTERNAL= + +# tag:proto tag:python-specific +macro NO_MYPY() { + DISABLE(PY_PROTO_MYPY_ENABLED) +} + +# tag:proto tag:python-specific +macro _PY_PROTO_CMD_BASE(File, Suf, Args...) { + .CMD=$PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_PATH --python_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $_PROTOC_FLAGS ${input;rootrel:File} ${output;hide;noauto;norel;nopath;noext;suf=$Suf:File} ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} $Args +} + +# tag:proto tag:python-specific +macro _PY_PROTO_CMD(File) { + .CMD=${cwd;rootdir;input:File} $_PY_PROTO_CMD_BASE($File _pb2.py $PY_PROTO_OPTS $PY_PROTO_OUTS $PY_PROTO_MYPY_PLUGIN) +} + +# tag:proto tag:python-specific +macro _PY_PROTO_CMD_INTERNAL(File) { + .CMD=${cwd;rootdir;input:File} $GEN_PY_PROTOS --suffixes $PY_PROTO_SUFFIXES $PY_PROTO_MYPY_SUFFIX -- $_PY_PROTO_CMD_BASE($File __int___pb2.py $PY_PROTO_OPTS $PY_PROTO_OUTS_INTERNAL ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__int___pb2.py:File} ${nopath;noext;suf=_pb2.py:File}"} $PY_PROTO_MYPY_PLUGIN_INTERNAL) +} + +# tag:proto tag:java-specific +JAVA_PROTO_RUNTIME= + +# tag:proto tag:java-specific +### @usage: USE_JAVALITE() +### Use protobuf-javalite for Java +macro USE_JAVALITE() { + SET(JAVA_PROTO_RUNTIME javalite) +} + +# tag:proto tag:java-specific +JAVA_PROTO_RUNTIME_VERSION = 3.21.12 +JAVA_PROTO_COMMON_VERSION = 1.12.0 +KOTLIN_PROTO=no +KOTLIN_PROTO_PEERS= +when ($JAVA_PROTO_RUNTIME == "javalite") { + _JAVA_PROTO_LITE_ARG=lite: + JAVA_PROTOBUF_PEERS=contrib/java/com/google/protobuf/protobuf-javalite/${JAVA_PROTO_RUNTIME_VERSION} ${KOTLIN_PROTO_PEERS} + JAVA_GRPC_STUB=contrib/java/io/grpc/grpc-stub/1.43.2 + JAVA_GRPC_PROTOBUF=contrib/java/io/grpc/grpc-protobuf/1.43.2 +} +otherwise { + _JAVA_PROTO_LITE_ARG= + JAVA_PROTOBUF_PEERS=contrib/java/com/google/protobuf/protobuf-java/${JAVA_PROTO_RUNTIME_VERSION} ${KOTLIN_PROTO_PEERS} + JAVA_GRPC_STUB=contrib/java/io/grpc/grpc-stub/1.43.2 + JAVA_GRPC_PROTOBUF=contrib/java/io/grpc/grpc-protobuf/1.43.2 +} + +KOTLIN_PROTO_FLAGS= +# tag:proto tag:java-specific +macro _JAVA_PROTO_CMD(File) { + .CMD=${cwd;rootdir;input:File} $YMAKE_PYTHON ${input:"build/scripts/tared_protoc.py"} --tar-output ${output;norel;nopath;noext;suf=.jsrc:File} --protoc-out-dir $ARCADIA_BUILD_ROOT/java_out $PROTOC -I=./$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_ROOT --java_out=${_JAVA_PROTO_LITE_ARG}$ARCADIA_BUILD_ROOT/java_out ${KOTLIN_PROTO_FLAGS} $_PROTOC_FLAGS ${input;rootrel:File} ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} $JAVA_PROTO_ARGS + .SEM=proto_files ${input;rootrel:File} ${hide;output:File.jsrc} +} + + +# tag:proto tag:perl-specific tag:deprecated +### @usage: XS_PROTO(InputProto Dir Outputs...) # deprecated +### +### Generate Perl code from protobuf. +### In order to use this macro one should predict all outputs protoc will emit from input_proto file and enlist those as outputs. +macro XS_PROTO(File, Dir, Outputs...) { + .CMD=$PROTOC -I=${ARCADIA_ROOT}/${Dir} ${pre=-I=:_PROTO__INCLUDE} -I=${ARCADIA_ROOT} -I=${PROTOBUF_PATH} --perlxs_out=${BINDIR} $_PROTOC_FLAGS ${input:File} ${hide;output:Outputs} ${induced_deps=h+cpp;hide;nopath;noext;suf=.pb.h:File} + PEERDIR(${Dir}) + ADDINCL(${ARCADIA_BUILD_ROOT}/${Dir}) +} + +# tag:proto tag:fbs +### @usage: PROTO2FBS(InputProto) +### +### Produce flatbuf schema out of protobuf description. +macro PROTO2FBS(File) { + .CMD=${cwd:BINDIR} ${tool:"contrib/tools/flatc"} -I . -I ${ARCADIA_ROOT} --proto ${input:File} ${output;hide;norel;noext:File.fbs} ${kv;hide:"p FBS"} ${kv;hide:"pc yellow"} && $MOVE_FILE ${BINDIR}/${nopath;noext:File.fbs} ${output;norel;noext:File.fbs} +} + +# tag:proto tag:python-specific +when ($PY_PROTOS_FOR == "yes") { + PEERDIR+=contrib/libs/protobuf/python +} + +# tag:python-specific +macro _PY_EVLOG_CMD_BASE(File, Suf, Args...) { + .CMD=$_PY_PROTO_CMD_BASE($File $Suf $Args) + .PEERDIR=library/cpp/eventlog/proto +} + +# tag:python-specific tag:proto +macro _PY_EVLOG_CMD(File) { + .CMD=${cwd;rootdir;input:File} $_PY_EVLOG_CMD_BASE($File _ev_pb2.py) +} + +# tag:python-specific tag:proto +macro _PY_EVLOG_CMD_INTERNAL(File) { + .CMD=${cwd;rootdir;input:File} $GEN_PY_PROTOS --suffixes $PY_EVLOG_SUFFIXES -- $_PY_EVLOG_CMD_BASE($File __int___ev_pb2.py ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__int___ev_pb2.py:File} ${nopath;noext;suf=_ev_pb2.py:File}"}) +} + +# tag:java-specific tag:proto +macro _JAVA_EVLOG_CMD(File) { + .CMD=$COPY_CMD ${input:File} ${output;nopath;noext;norel;suf=_ev.proto:File} ${kv;hide:"p EV"} ${kv;hide:"pc yellow"} + .PEERDIR=library/cpp/eventlog/proto +} + +# tag:proto tag:grpc +_GRPC_ENABLED=no + +# tag:proto tag:grpc +### @usage: GRPC() +### +### Emit GRPC code for all .proto files in a PROTO_LIBRARY. +### This works for all available PROTO_LIBRARY versions (C++, Python 2.x, Python 3.x, Java and Go). +macro GRPC() { + ENABLE(_GRPC_ENABLED) + + # C++ + CPP_PROTO_PLUGIN2(grpc_cpp contrib/tools/protoc/plugins/grpc_cpp .grpc.pb.cc .grpc.pb.h DEPS contrib/libs/grpc) + + # Python + PY_PROTO_PLUGIN(grpc_py _pb2_grpc.py contrib/tools/protoc/plugins/grpc_python DEPS contrib/python/grpcio) + + # Java + JAVA_PROTO_PLUGIN(grpc_java contrib/tools/protoc/plugins/grpc_java DEPS $JAVA_GRPC_STUB $JAVA_GRPC_PROTOBUF) + SET_APPEND(JAVA_PROTOBUF_PEERS contrib/java/javax/annotation/javax.annotation-api/1.3.1) +} + +macro GO_PROTO_USE_V2() { + ENABLE(GO_PROTO_V2) +} + +# tag:proto +### @usage: RESOLVE_PROTO() +### +### Enable include resolving within UNIONs and let system .proto being resolved +### among .proto/.gztproto imports +### +### Note: it is currently impossible to enable resolving only for .proto, so resolving is enabled for all supported files +### also we only add ADDINCL for stock protobuf. So use this macro with care: it may cause resolving problems those are +### to be addressed by either ADDINCLs or marking them as TEXT. Please contact devtools for details. +macro RESOLVE_PROTO() { + SET(DONT_RESOLVE_INCLUDES no) + ADDINCL(FOR proto $PROTOBUF_PATH) +} + +# tag:python-specific tag:proto tag:deprecated +### @usage GENERATE_PY_PROTOS(ProtoFiles...) # deprecated +### +### Generate python bindings for protobuf files. +### Macro is obsolete and not recommended for use! +macro GENERATE_PY_PROTOS(FILES...) { + foreach (FILE : $FILES) { + _PY_PROTO_CMD($FILE) + } +} + +# tag:python-specific tag:proto +macro _GENERATE_PY_PROTOS_INTERNAL(FILES...) { + foreach (FILE : $FILES) { + _PY_PROTO_CMD_INTERNAL($FILE) + } +} + +# tag:python-specific +macro _GENERATE_PY_EVS_INTERNAL(FILES...) { + foreach (FILE : $FILES) { + _PY_EVLOG_CMD_INTERNAL($FILE) + } +} + +### @usage: LIST_PROTO([TO list.proto] Files...) # deprecated +### +### Create list of .proto files in a list-file (should be .proto, files.proto by default) +### with original .proto-files as list's dependencies. +### +### This allows to process files listed, passing list as an argument to the processor +### +### TODO: proper implementation needed +macro LIST_PROTO(TO="files.proto", Files...) { + .CMD=$YMAKE_PYTHON3 ${input:"build/scripts/list.py"} ${Files} ${input;hide:Files} ${stdout;output;noauto:TO} ${output_include;from_input;hide:Files} && $YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${output;noext;suf=.pb.h:TO} + _EXPOSE(${TO}) +} + +# tag:proto +macro _PROTO_DESC_CMD(File) { + .CMD=${cwd;rootdir;input:File} $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_PATH --descriptor_set_out=${output;suf=.desc:File} --include_source_info $_PROTOC_FLAGS ${input;rootrel:File} +} + +_PROTO_DESC_MERGE_CMD=$YMAKE_PYTHON ${input:"build/scripts/merge_files.py"} $TARGET $AUTO_INPUT ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"} +_PROTO_DESC_MERGE_PEERS_CMD=$YMAKE_PYTHON ${input:"build/scripts/merge_files.py"} $TARGET $PEERS $SRCS_GLOBAL ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"} + +NEED_GOOGLE_PROTO_PEERDIRS=yes + +CPP_PROTO_LIBRARY_SEM=$CPP_LIBRARY_SEM + +JAVA_PROTO_LIBRARY_SEM=$BUILD_PROTO_JAR_SEM $_GRADLE_EXPORT_PUBLISHING_SEM + +# tag:proto +### @usage: PROTO_LIBRARY() +### +### Build some varian of protocol buffers library. +### +### The particular variant is selected based on where PEERDIR to PROTO_LIBRARY comes from. +### +### Now supported 5 variants: C++, Java, Python 2.x, Python 3.x and Go. +### When PEERDIR comes from module for particular language appropriate variant is selected. +### PROTO_LIBRARY also supports emission of GRPC code if GRPC() macro is specified. +### Notes: +### - Python versions emit C++ code in addition to Python as optimization. +### - In some PROTO_LIBRARY-es Java or Python versions are excluded via EXCLUDE_TAGS macros due to incompatibilities. +### - Use from DEPENDS or BUNDLE is not allowed +### +### Documentation: https://wiki.yandex-team.ru/yatool/proto_library/ +### +### See: [GRPC()](#macro_GRPC), [OPTIMIZE_PY_PROTOS()](#macro_OPTIMIZE_PY_PROTOS), [INCLUDE_TAGS()](#macro_INCLUDE_TAGS), [EXCLUDE_TAGS()](#macro_EXCLUDE_TAGS) +multimodule PROTO_LIBRARY { + module CPP_PROTO : LIBRARY { + .ALLOWED=_EXPOSE LIST_PROTO + # TODO(svidyuk): think about marker which forces semantics inheritance + .SEM=CPP_PROTO_LIBRARY_SEM + FORCE_COVERAGE_DISABLED=yes + ENABLE(CPP_PROTO) + ENABLE(GEN_PROTO) + NO_CLANG_TIDY() + SET(PEERDIR_TAGS CPP_PROTO) + + when ($BUILD_PROTO_AS_EVLOG == "yes" && $USE_VANILLA_PROTOC == "yes") { + _OK=no + } + ASSERT(_OK BUILD_PROTO_AS_EVLOG and USE_VANILLA_PROTOC are incompatible yet) + + MODULE_SUFFIX=$_CPP_PROTO_MODULE_SUFFIX + MODULE_PREFIX=$_CPP_PROTO_MODULE_PREFIX + + when ($_COMMON_GOOGLE_APIS != "None") { + PEERDIR += contrib/libs/googleapis-common-protos + } + } + + module JAVA_PROTO: EXTERNAL_JAVA_LIBRARY { + .EXTS=.jsrc + .ALLOWED=GRPC + .SEM=JAVA_PROTO_LIBRARY_SEM + SET(PEERDIR_TAGS JAVA_PROTO) + ENABLE(JAVA_PROTO) + PEERDIR+=$JAVA_PROTOBUF_PEERS + + when ($KOTLIN_PROTO == "yes") { + KOTLIN_PROTO_PEERS=contrib/java/com/google/protobuf/protobuf-kotlin/${JAVA_PROTO_RUNTIME_VERSION} + KOTLIN_PROTO_FLAGS=--kotlin_out=$ARCADIA_BUILD_ROOT/java_out + } + when ($GRADLE_EXPORT_PUBLISHING == "yes") { + _GRADLE_EXPORT_PUBLISHING_SEM=$_DO_GRADLE_EXPORT_PUBLISHING_SEM + } + + .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_SKIFF CPP_PROTO_PLUGIN2 PY_PROTO_PLUGIN YMAPS_SPROTO RESOURCE + ADDINCL(FOR proto $PROTOBUF_PATH) + + when ($_COMMON_GOOGLE_APIS != "None") { + PEERDIR += contrib/java/com/google/api/grpc/proto-google-common-protos/${JAVA_PROTO_COMMON_VERSION} + ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos + } + } + + module PY_PROTO: PY2_LIBRARY { + .ALIASES=SRCS=PY_SRCS + .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS + .PEERDIRSELF=CPP_PROTO + .SEM=IGNORED + SET(PEERDIR_TAGS PY2 PY_PROTO) + ENABLE(PY_PROTO) + OPTIMIZE_PY_PROTOS() + OBJ_SUF=.py2 + # Can not use NO_LINT(), because is not allowed outside of contrib directory + SET(_NO_LINT_VALUE none_internal) + + when ($_COMMON_GOOGLE_APIS != "None") { + PEERDIR += contrib/libs/googleapis-common-protos + } + + _IGNORE_SELF_PEERS= + _CPP_PROTO_LIBRARY=${MODDIR}/$_CPP_PROTO_MODULE_PREFIX$REALPRJNAME$_CPP_PROTO_MODULE_SUFFIX + when ($OPTIMIZE_PY_PROTOS_FLAG == "no") { + _IGNORE_PEERDIRSELF=CPP_PROTO + } + SET_APPEND(_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL $_CPP_PROTO_LIBRARY) + } + + module PY3_PROTO: PY3_LIBRARY { + .ALIASES=SRCS=PY_SRCS + .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS + .PEERDIRSELF=CPP_PROTO + .SEM=IGNORED + SET(PEERDIR_TAGS PY3 PY3_PROTO) + ENABLE(PY3_PROTO) + OPTIMIZE_PY_PROTOS() + OBJ_SUF=.py3 + # Can not use NO_LINT(), because is not allowed outside of contrib directory + SET(_NO_LINT_VALUE none_internal) + + when ($_COMMON_GOOGLE_APIS != "None") { + PEERDIR += contrib/libs/googleapis-common-protos + } + + _IGNORE_SELF_PEERS= + _CPP_PROTO_LIBRARY=${MODDIR}/$_CPP_PROTO_MODULE_PREFIX$REALPRJNAME$_CPP_PROTO_MODULE_SUFFIX + when ($OPTIMIZE_PY_PROTOS_FLAG == "no") { + _IGNORE_PEERDIRSELF=CPP_PROTO + } + SET_APPEND(_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL $_CPP_PROTO_LIBRARY) + } + + module GO_PROTO: GO_LIBRARY { + .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO + .SEM=IGNORED + SET(PEERDIR_TAGS GO GO_PROTO) + ENABLE(GO_PROTO) + + when ($_COMMON_GOOGLE_APIS == "None") { + } + elsewhen ($_COMMON_GOOGLE_APIS == "") { + PEERDIR += $_GO_COMMON_GOOGLE_APIS + ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos + } + otherwise { + PEERDIR += $_COMMON_GOOGLE_APIS + ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos + } + } + + module DESC_PROTO: _BARE_UNIT { + .CMD=_PROTO_DESC_MERGE_CMD + .SEM=IGNORED + .EXTS=.desc + .NODE_TYPE=Library + .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO RESOURCE GO_PROTO_PLUGIN GRPC + + SET(PEERDIR_TAGS DESC_PROTO) + ENABLE(DESC_PROTO) + MODULE_SUFFIX=.self.protodesc + SET(MODULE_TYPE LIBRARY) + + MACRO_ALIAS(EVLOG_CMD _PROTO_DESC_CMD) + MACRO_ALIAS(PROTO_CMD _PROTO_DESC_CMD) + + when ($_COMMON_GOOGLE_APIS != "None") { + PEERDIR += contrib/libs/googleapis-common-protos + } + + when ($NEED_GOOGLE_PROTO_PEERDIRS == "yes") { + when ($USE_VANILLA_PROTOC == "yes") { + PEERDIR += contrib/libs/protobuf_std/builtin_proto/protos_from_protobuf + } + otherwise { + PEERDIR += contrib/libs/protobuf/builtin_proto/protos_from_protoc + } + } + } +} + +module PROTO_DESCRIPTIONS: _BARE_UNIT { + .CMD=_PROTO_DESC_MERGE_PEERS_CMD + .PEERDIR_POLICY=as_build_from + .NODE_TYPE=Library + .RESTRICTED=SRCS + .FINAL_TARGET=yes + + SET(PEERDIR_TAGS DESC_PROTO) + SET(MODULE_SUFFIX .protodesc) + SET(MODULE_TYPE PROTO_DESCRIPTIONS) +} + +module PROTO_REGISTRY: PROTO_DESCRIPTIONS { + SET(MODULE_TYPE PROTO_REGISTRY) +} diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 02e0bf6306..639c412074 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -38,6 +38,9 @@ FORCE_COVERAGE_DISABLED= AUTOCHECK_BALANCING_OPTIONS= SO_OUTPUTS=no +MODULE_PREFIX= +MODULE_SUFFIX= + @import "${CONF_ROOT}/conf/settings.conf" @import "${CONF_ROOT}/conf/opensource.conf" @import "${CONF_ROOT}/conf/rules.conf" @@ -47,6 +50,7 @@ SO_OUTPUTS=no @import "${CONF_ROOT}/conf/docs.conf" @import "${CONF_ROOT}/conf/swig.conf" @import "${CONF_ROOT}/conf/perl.conf" +@import "${CONF_ROOT}/conf/proto.conf" @import "${CONF_ROOT}/conf/project_specific/other.conf" @import "${CONF_ROOT}/conf/project_specific/uservices.conf" @@ -230,15 +234,6 @@ SFDL_TMP_OUT= ${output;tmp:SRC.tmp} # tag:tool-specific ARCH_TOOL=${tool:"tools/archiver"} -# tag:tool-specific tag:proto -PROTOC=${tool:"contrib/tools/protoc"} -PROTOC_STYLEGUIDE_OUT=--cpp_styleguide_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE -PROTOC_PLUGIN_STYLEGUIDE=--plugin=protoc-gen-cpp_styleguide=${tool:"contrib/tools/protoc/plugins/cpp_styleguide"} -PROTOBUF_PATH=${ARCADIA_ROOT}/contrib/libs/protobuf/src -USE_VANILLA_PROTOC=no -PROTOC_TRANSITIVE_HEADERS=yes -_PROTOC_FLAGS= - # tag:tool-specific FML_TOOL=${tool:"tools/relev_fml_codegen"} FML_UNUSED_TOOL=${tool:"tools/relev_fml_unused"} @@ -349,44 +344,6 @@ macro CHECK_DEPENDENT_DIRS(TYPE, ALL?"UNUSED":"", PEERDIRS?"PEERDIRS":"ALL", RES SET_APPEND(CHECK_DEPENDENT_DIRS_TYPES $TYPE) } -# tag:proto -PY_PROTOS_FOR=no -BUILD_PROTO_AS_EVLOG=no - -# tag:proto -PROTO_NAMESPACE= - -# tag:proto tag:python-specific -GEN_PY_PROTOS=$YMAKE_PYTHON ${input:"build/scripts/gen_py_protos.py"} - -# tag:proto tag:cpp-specific -PROTO_HEADER_EXTS=.pb.h -CPP_PROTO_CMDLINE= -CPP_PROTO_OPTS= -CPP_PROTO_OUTS= -CPP_PROTO_OUTS_SEM= -CPP_PROTO_SUFFIXES=.pb.h .pb.cc -CPP_PROTO_PLUGINS= - -# tag:proto tag:cpp-specific -CPP_EV_OPTS=--plugin=protoc-gen-event2cpp=${tool:"tools/event2cpp"} --event2cpp_out=$ARCADIA_BUILD_ROOT -I=$ARCADIA_ROOT/library/cpp/eventlog -CPP_EV_OUTS= -CPP_EV_OUTS_SEM= - -# tag:proto tag:python-specific -PY_PROTO_OPTS= -PY_PROTO_OUTS= -PY_PROTO_OUTS_INTERNAL= -PY_PROTO_DEPS= -PY_PROTO_SUFFIXES=_pb2.py -PY_EVLOG_SUFFIXES=_ev_pb2.py - -# tag:proto tag:java-specific -JAVA_PROTO_ARGS= - -# tag:proto tag:python-specific -OPTIMIZE_PY_PROTOS_FLAG=no - macro _RESOURCE_SEM(INPUTS[], KEYS[]) { SET(RESOURCE_OUTPUT ${hash:INPUTS}.cpp) .SEM=resources ${output;global:RESOURCE_OUTPUT} INPUTS ${input:INPUTS} KEYS $KEYS ${tool;hide:"tools/rescompiler/bin"} @@ -420,463 +377,6 @@ macro RESOURCE(Args...) { PEERDIR(library/cpp/resource) } -# tag:internal -### @usage: _ORDER_ADDINCL([BUILD ...] [SOURCE ...] Args...) # internal -### -### Order and filter ADDINCLs (Args - is intentionally omitted in ADDINCL macro) -macro _ORDER_ADDINCL(BUILD[], SOURCE[], ARGS...) { - ADDINCL($BUILD $SOURCE) -} - -# tag:proto -### @usage: PROTO_ADDINCL([GLOBAL] [WITH_GEN] Path) -### -### This macro introduces proper ADDINCLs for .proto-files found in sources and -### .cpp/.h generated files, supplying them to appropriate commands and allowing -### proper dependency resolution at configure-time. -### -### Note: you normally shouldn't use this macro. ADDINCLs should be sent to user -### from dependency via PROTO_NAMESPACE macro -macro PROTO_ADDINCL(GLOBAL?"GLOBAL":"", Path, WITH_GEN?"BUILD":"") { - _ORDER_ADDINCL($WITH_GEN $GLOBAL FOR proto ${ARCADIA_BUILD_ROOT}/$Path SOURCE $GLOBAL FOR proto ${ARCADIA_ROOT}/$Path) - ADDINCL($GLOBAL ${ARCADIA_BUILD_ROOT}/$Path) -} - -# tag:proto -### @usage: PROTO_NAMESPACE([WITH_GEN] Namespace) -### -### Defines protobuf namespace (import/export path prefix) which should be used for imports and -### which defines output path for .proto generation. -### -### For proper importing and configure-time dependency management it sets ADDINCLs -### for both .cpp headers includes and .proto imports. If .proto expected to be used outside of the -### processing module use GLOBAL to send proper ADDINCLs to all (transitive) users. PEERDIR to -### PROTO_LIBRARY with PROTO_NAMESPACE() is enough at user side to correctly use the library. -### If generated .proto files are going to be used for building a module than use of WITH_GEN -### parameter will add appropriate dir from the build root for .proto files search. -macro PROTO_NAMESPACE(GLOBAL?"GLOBAL":"", WITH_GEN?"WITH_GEN":"", Namespace) { - SET(PROTO_NAMESPACE $Namespace) - PROTO_ADDINCL(GLOBAL $WITH_GEN $Namespace) -} - -# tag:proto -### @usage PROTOC_FATAL_WARNINGS() -### -### Treat protoc warnings as fatal errors that break the build, for example, unused imports -### Adds `--fatal_warnings` argument to protoc -macro PROTOC_FATAL_WARNINGS() { - SET_APPEND(_PROTOC_FLAGS --fatal_warnings) -} - -# tag:proto tag:internal tag:python-specific -### @usage: OPTIMIZE_PY_PROTOS() # internal -### -### Enable Python proto optimization by embedding corresponding C++ code into binary. -### Python protobuf runtime will use C++ implementation instead of Python one if former is available. -### This is default mode for most PROTO_LIBRARY's and PY2_LIBRARY's, some system ones being exceptions. -macro OPTIMIZE_PY_PROTOS() { - SET(OPTIMIZE_PY_PROTOS_FLAG yes) -} - -# tag:proto tag:python-specific -### @usage: NO_OPTIMIZE_PY_PROTOS() -### -### Disable Python proto optimization using embedding corresponding C++ code into binary. -### Python protobuf runtime will use C++ implementation instead of Python one if former is available. -### This is default mode only for some system libraries. -macro NO_OPTIMIZE_PY_PROTOS() { - SET(OPTIMIZE_PY_PROTOS_FLAG no) -} - -# tag:proto tag:python-specific -macro _PROTO_PLUGIN_ARGS_BASE(Name, Tool, OutParm...) { - .CMD=--plugin=protoc-gen-${Name}=\${tool:"$Tool"} --${Name}_out=$ARCADIA_BUILD_ROOT/\$PROTO_NAMESPACE ${pre=--${Name}_opt=:OutParm} - .SEM=IGNORED -} - -# tag:proto tag:python-specific -macro _ADD_PY_PROTO_OUT(Suf) { - SET_APPEND(PY_PROTO_OUTS \${output;hide;noauto;norel;nopath;noext;suf=$Suf:File}) - SET_APPEND(PY_PROTO_OUTS_INTERNAL \${output;hide;noauto;norel;nopath;noext;suf=__int__$Suf:File} \${hide;kv:"ext_out_name_for_\${nopath;noext;suf=__int__$Suf:File} \${nopath;noext;suf=$Suf:File}"}) - # XXX fix variable expansion in plugins - SET(PY_PROTO_SUFFIXES $PY_PROTO_SUFFIXES $Suf) -} - -# tag:proto tag:python-specific -### @usage: PY_PROTO_PLUGIN(Name Ext Tool DEPS <Dependencies>) -### -### Define protoc plugin for python with given Name that emits extra output with provided Extension -### using Tool. Extra dependencies are passed via DEPS -macro PY_PROTO_PLUGIN(NAME, EXT, TOOL, DEPS[]) { - SET_APPEND(PY_PROTO_OPTS $_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL)) - _ADD_PY_PROTO_OUT($EXT) - # XXX fix variable expansion in plugins - SET(PY_PROTO_DEPS $PY_PROTO_DEPS $DEPS) -} - -# tag:proto tag:python-specific -### @usage: PY_PROTO_PLUGIN2(Name Ext1 Ext2 Tool DEPS <Dependencies>) -### -### Define protoc plugin for python with given Name that emits 2 extra outputs with provided Extensions -### using Tool. Extra dependencies are passed via DEPS -macro PY_PROTO_PLUGIN2(NAME, EXT1, EXT2, TOOL, DEPS[]) { - PY_PROTO_PLUGIN($NAME $EXT1 $TOOL DEPS $DEPS) - _ADD_PY_PROTO_OUT($EXT2) -} - -# tag:proto tag:java-specific -macro _JAVA_PROTO_PLUGIN_ARGS_BASE(Name, Tool, OutParm...) { - .CMD=--plugin=protoc-gen-${Name}=\${tool:"$Tool"} --${Name}_out=$ARCADIA_BUILD_ROOT/java_out -} - -# tag:proto tag:java-specific -### @usage: JAVA_PROTO_PLUGIN(Name Tool DEPS <Dependencies>) -### -### Define protoc plugin for Java with given Name that emits extra outputs -### using Tool. Extra dependencies are passed via DEPS -macro JAVA_PROTO_PLUGIN(NAME, TOOL, DEPS[]) { - SET_APPEND(JAVA_PROTO_ARGS $_JAVA_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL)) - # XXX fix variable expansion in plugins - SET(JAVA_PROTOBUF_PEERS $JAVA_PROTOBUF_PEERS $DEPS) -} - -macro _ADD_SEM_PROP_IF_NON_EMPTY(Prop, Args...) { - .SEM=${pre=&& set_target_property $Prop :Args} -} - -# tag:proto tag:cpp-specific -macro _ADD_CPP_PROTO_OUT(Suf) { - .SEM=append_target_property PROTOC_EXTRA_OUTS $Suf ${output;hide;suf=.o:Suf} $_ADD_SEM_PROP_IF_NON_EMPTY(PROTO_NAMESPACE $PROTO_NAMESPACE) - SET_APPEND(CPP_PROTO_OUTS \${output;hide;norel;nopath;noext;suf=$Suf:File}) - - # XXX fix variable expansion in plugins - SET(CPP_PROTO_SUFFIXES $CPP_PROTO_SUFFIXES $Suf) -} - -# tag:proto tag:cpp-specific -HAS_CPP_PROTOBUF_PEERS=no -CPP_PROTOBUF_PEERS= - -# tag:proto tag:cpp-specific -### @usage: CPP_PROTO_PLUGIN0(Name Tool DEPS <Dependencies>) -### -### Define protoc plugin for C++ with given Name that emits code into regular outputs -### using Tool. Extra dependencies are passed via DEPS. -macro CPP_PROTO_PLUGIN0(NAME, TOOL, DEPS[]) { - .SEM=target_proto_plugin $NAME ${tool;rootrel:TOOL} ${output;hide;suf=.fake.o:NAME} - SET_APPEND(CPP_PROTO_OPTS $_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL)) - - # XXX fix variable expansion in plugins - ENABLE(HAS_CPP_PROTOBUF_PEERS) - SET(CPP_PROTOBUF_PEERS $CPP_PROTOBUF_PEERS $DEPS) -} - -# tag:proto tag:cpp-specific -### @usage: CPP_PROTO_PLUGIN(Name Tool Suf DEPS <Dependencies>) -### -### Define protoc plugin for C++ with given Name that emits code into 1 extra output -### using Tool. Extra dependencies are passed via DEPS. -macro CPP_PROTO_PLUGIN(NAME, TOOL, SUF, DEPS[]) { - CPP_PROTO_PLUGIN0($NAME $TOOL DEPS $DEPS) - - _ADD_CPP_PROTO_OUT($SUF) -} - -# tag:proto tag:cpp-specific -### @usage: CPP_PROTO_PLUGIN2(Name Tool Suf1 Suf2 DEPS <Dependencies>) -### -### Define protoc plugin for C++ with given Name that emits code into 2 extra outputs -### using Tool. Extra dependencies are passed via DEPS. -macro CPP_PROTO_PLUGIN2(NAME, TOOL, SUF1, SUF2, DEPS[]) { - CPP_PROTO_PLUGIN($NAME $TOOL $SUF1 DEPS $DEPS) - - _ADD_CPP_PROTO_OUT($SUF2) - SET_APPEND(CPP_PROTO_OUTS_SEM \${output;hide;norel;nopath;noext;suf=$SUF2:File}) -} - -# tag:proto -### @usage: USE_SKIFF() #wip, do not use -### -### Use mapreduce/yt/skiff_proto/plugin for C++ -macro USE_SKIFF() { - # Move extra includes to INDUCED_DEPS macro in mapreduce/yt/skiff_proto/plugin/ya.make - CPP_PROTO_PLUGIN0(skiff mapreduce/yt/skiff_proto/plugin DEPS mapreduce/yt/skiff_proto/lib) -} - -# tag:go-specific -_GO_COMMON_GOOGLE_APIS = \ -vendor/google.golang.org/genproto/googleapis/api/annotations \ -vendor/google.golang.org/genproto/googleapis/api/configchange \ -vendor/google.golang.org/genproto/googleapis/api/distribution \ -vendor/google.golang.org/genproto/googleapis/api/expr/v1alpha1 \ -vendor/google.golang.org/genproto/googleapis/api/expr/v1beta1 \ -vendor/google.golang.org/genproto/googleapis/api/httpbody \ -vendor/google.golang.org/genproto/googleapis/api/label \ -vendor/google.golang.org/genproto/googleapis/api/metric \ -vendor/google.golang.org/genproto/googleapis/api/monitoredres \ -vendor/google.golang.org/genproto/googleapis/api/serviceconfig \ -vendor/google.golang.org/genproto/googleapis/api/servicecontrol/v1 \ -vendor/google.golang.org/genproto/googleapis/api/servicemanagement/v1 \ -vendor/google.golang.org/genproto/googleapis/iam/admin/v1 \ -vendor/google.golang.org/genproto/googleapis/iam/credentials/v1 \ -vendor/google.golang.org/genproto/googleapis/iam/v1 \ -vendor/google.golang.org/genproto/googleapis/iam/v1/logging \ -vendor/google.golang.org/genproto/googleapis/logging/type \ -vendor/google.golang.org/genproto/googleapis/logging/v2 \ -vendor/google.golang.org/genproto/googleapis/rpc/code \ -vendor/google.golang.org/genproto/googleapis/rpc/errdetails \ -vendor/google.golang.org/genproto/googleapis/rpc/status \ -vendor/google.golang.org/genproto/googleapis/type/calendarperiod \ -vendor/google.golang.org/genproto/googleapis/type/color \ -vendor/google.golang.org/genproto/googleapis/type/date \ -vendor/google.golang.org/genproto/googleapis/type/dayofweek \ -vendor/google.golang.org/genproto/googleapis/type/expr \ -vendor/google.golang.org/genproto/googleapis/type/fraction \ -vendor/google.golang.org/genproto/googleapis/type/latlng \ -vendor/google.golang.org/genproto/googleapis/type/money \ -vendor/google.golang.org/genproto/googleapis/type/postaladdress \ -vendor/google.golang.org/genproto/googleapis/type/quaternion \ -vendor/google.golang.org/genproto/googleapis/type/timeofday - -# tag:go-specific -_COMMON_GOOGLE_APIS=None - -# tag:go-specific -### @usage: USE_COMMON_GOOGLE_APIS([apis...]) -### -### Use common-google-apis library set. Pass optional apis list to be used or use them all. -### This macro is properly handled for all languages including Go, where apis come in -## pregenerated form. In other languages apis are generated from sources in Arcadia. -macro USE_COMMON_GOOGLE_APIS(APIS...) { - SET(_COMMON_GOOGLE_APIS ${pre=vendor/google.golang.org/genproto/googleapis/:APIS}) -} - -# tag:go-specific tag:proto -GO_PROTO_GEN_PLUGINS= -GO_PROTO_OPTS= -GO_PROTO_OUTS= -GO_PROTO_GRPC_OPTS= -GO_PROTO_GRPC_OUTS= -GO_PROTO_V2=no -_GO_PROTO_CHECK_OUTPUT= - -# tag:go-specific tag:proto -GO_PROTOBUF_IMPORTS=\ - ${GOSTD}/sync \ - ${GOSTD}/reflect \ - ${GOSTD}/fmt \ - ${GOSTD}/math \ - vendor/github.com/golang/protobuf/proto \ - vendor/google.golang.org/protobuf/types/descriptorpb \ - vendor/google.golang.org/protobuf/runtime/protoimpl \ - vendor/google.golang.org/protobuf/runtime/protoiface \ - vendor/google.golang.org/protobuf/reflect/protoreflect - -# tag:go-specific tag:proto -GO_PROTOBUF_WELLKNOWN_TYPES=\ - vendor/google.golang.org/protobuf/types/known/anypb \ - vendor/google.golang.org/protobuf/types/known/apipb \ - vendor/google.golang.org/protobuf/types/known/durationpb \ - vendor/google.golang.org/protobuf/types/known/emptypb \ - vendor/google.golang.org/protobuf/types/known/fieldmaskpb \ - vendor/google.golang.org/protobuf/types/known/sourcecontextpb \ - vendor/google.golang.org/protobuf/types/known/structpb \ - vendor/google.golang.org/protobuf/types/known/timestamppb \ - vendor/google.golang.org/protobuf/types/known/typepb \ - vendor/google.golang.org/protobuf/types/known/wrapperspb - -# tag:go-specific tag:proto -### @usage: GO_PROTO_PLUGIN(Name Ext Tool [DEPS dependencies...]) -### -### Define protoc plugin for GO with given Name that emits extra output with provided extension -### Ext using Tool. Extra dependencies are passed via DEPS. -macro GO_PROTO_PLUGIN(NAME, EXT, TOOL, DEPS[]) { - SET_APPEND(GO_PROTO_OPTS $_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL)) - SET_APPEND(GO_PROTO_OUTS \${output;hide;noauto;norel;nopath;noext;suf=$EXT:File}) - PEERDIR(${DEPS}) -} - -# tag:go-specific tag:proto -GO_PROTO_CMDLINE=${cwd;rootdir;input:File} $YMAKE_PYTHON3 ${input:"build/scripts/go_proto_wrapper.py"} --arcadia-prefix $GO_ARCADIA_PROJECT_PREFIX --contrib-prefix $GO_CONTRIB_PROJECT_PREFIX --namespace ./$PROTO_NAMESPACE $_GO_PROTO_CHECK_OUTPUT --proto ${input;rootrel:File} -- $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_PATH $_PROTOC_FLAGS - -# tag:go-specific tag:proto -macro _GO_PROTO_CMD_IMPL(File, OPTS...) { - .CMD=$GO_PROTO_CMDLINE $OPTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} - .PEERDIR=${GO_PROTOBUF_IMPORTS} ${GO_PROTOBUF_WELLKNOWN_TYPES} - .ADDINCL=FOR proto contrib/libs/protobuf/src -} - -# tag:go-specific tag:proto -macro _GO_PROTO_CMD(File) { - .CMD=$_GO_PROTO_CMD_IMPL($File $GO_PROTO_OPTS $GO_PROTO_OUTS) -} - -# tag:proto -### @usage: YP_PROTO_YSON(Files... OUT_OPTS Opts...) -### -### Generate .yson.go from .proto using yp/go/yson/internal/proto-yson-gen/cmd/proto-yson-gen -macro YP_PROTO_YSON(OUT_OPTS[], Files...) { - .CMD=${cwd:BINDIR} $PROTOC --plugin=protoc-gen-custom=${tool:"yp/go/yson/internal/proto-yson-gen/cmd/proto-yson-gen"} -I=${ARCADIA_ROOT}/${PROTO_NAMESPACE} ${pre=-I=:_PROTO__INCLUDE} -I=${ARCADIA_ROOT} --custom_out="$OUT_OPTS paths=base_name:." --custom_opt="goroot=${GO_TOOLS_ROOT}" $_PROTOC_FLAGS ${input:Files} ${output;hide;noauto;nopath;noext;suf=.yson.go:Files} - .ADDINCL=FOR proto ${ARCADIA_ROOT}/${MODDIR} FOR proto ${ARCADIA_ROOT}/${GO_TEST_IMPORT_PATH} FOR proto yt ${ARCADIA_BUILD_ROOT}/yt FOR proto contrib/libs/protobuf/src - .PEERDIR=$GOSTD/strings $GOSTD/fmt $GOSTD/errors $GOSTD/encoding/json library/go/core/xerrors yt/go/yson yt/go/yterrors yp/go/yson/ytypes contrib/libs/protobuf - -} - -_SEM_CPP_PROTO_CMD=target_proto_messages PRIVATE ${input:File} $CPP_PROTO_OUTS_SEM ${output;hide;suf=${OBJ_SUF}.pb.o:File} ${hide;tool:"contrib/tools/protoc/bin"} ${hide;tool:"contrib/tools/protoc/plugins/cpp_styleguide"} \ - && set_global_flags COMMON_PROTOC_FLAGS \ - && target_proto_outs --cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $PROTOC_STYLEGUIDE_OUT \ - && target_proto_addincls ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE ${_PROTO__INCLUDE} $ARCADIA_BUILD_ROOT $PROTOBUF_PATH - -# tag:proto -macro _CPP_PROTO_CMD(File) { - .CMD=$CPP_PROTO_CMDLINE $CPP_PROTO_OPTS $CPP_PROTO_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} - .SEM=$_SEM_CPP_PROTO_CMD - .PEERDIR=contrib/libs/protobuf -} - -# tag:proto -macro _CPP_VANILLA_PROTO_CMD(File) { - .CMD=$CPP_PROTO_CMDLINE $CPP_PROTO_OPTS $CPP_PROTO_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} - .PEERDIR=contrib/libs/protobuf_std -} - -_SEM_CPP_EV_CMD=target_ev_messages PRIVATE ${input:File} $CPP_EV_OUTS_SEM ${output;hide;suf=${OBJ_SUF}.pb.o:File} ${hide;tool:"tools/event2cpp/bin"} ${hide;tool:"contrib/tools/protoc/bin"} ${hide;tool:"contrib/tools/protoc/plugins/cpp_styleguide"} \ - && set_global_flags COMMON_PROTOC_FLAGS \ - && target_proto_outs --cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $PROTOC_STYLEGUIDE_OUT --event2cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE \ - && target_proto_addincls ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE ${_PROTO__INCLUDE} $ARCADIA_BUILD_ROOT $PROTOBUF_PATH $ARCADIA_ROOT/library/cpp/eventlog - -# tag:proto -macro _CPP_EVLOG_CMD(File) { - .CMD=$CPP_PROTO_CMDLINE $CPP_EV_OPTS $CPP_EV_OUTS ${kv;hide:"p EV"} ${kv;hide:"pc yellow"} - .SEM=$_SEM_CPP_EV_CMD - .PEERDIR=library/cpp/eventlog contrib/libs/protobuf -} - -# tag:proto -macro _CPP_PROTO_EVLOG_CMD(File) { - # process .proto as .ev - .CMD=$CPP_PROTO_CMDLINE $CPP_EV_OPTS $CPP_PROTO_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} - .PEERDIR=library/cpp/eventlog contrib/libs/protobuf -} - -# tag:proto -macro _CPP_CFGPROTO_CMD(File) { - # keep extension in output just as in EV: this is hard-coded behaviour of protoc for non-.proto extensions - .CMD=$CPP_PROTO_CMDLINE --plugin=protoc-gen-config=${tool:"library/cpp/proto_config/plugin"} --config_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $CPP_EV_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} - .PEERDIR=library/cpp/proto_config/codegen library/cpp/proto_config/protos contrib/libs/protobuf -} - -# tag:proto -PY_PROTO_MYPY_ENABLED=yes -PY_PROTO_MYPY_PLUGIN_BASE=--plugin=protoc-gen-mypy=${tool:"contrib/python/mypy-protobuf/bin/protoc-gen-mypy"} --mypy_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE - -# tag:proto tag:python-specific -PY_PROTO_MYPY_SUFFIX= -PY_PROTO_MYPY_PLUGIN= -PY_PROTO_MYPY_PLUGIN_INTERNAL= - -# tag:proto tag:python-specific -macro NO_MYPY() { - DISABLE(PY_PROTO_MYPY_ENABLED) -} - -# tag:proto tag:python-specific -macro _PY_PROTO_CMD_BASE(File, Suf, Args...) { - .CMD=$PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_PATH --python_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $_PROTOC_FLAGS ${input;rootrel:File} ${output;hide;noauto;norel;nopath;noext;suf=$Suf:File} ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} $Args -} - -# tag:proto tag:python-specific -macro _PY_PROTO_CMD(File) { - .CMD=${cwd;rootdir;input:File} $_PY_PROTO_CMD_BASE($File _pb2.py $PY_PROTO_OPTS $PY_PROTO_OUTS $PY_PROTO_MYPY_PLUGIN) -} - -# tag:proto tag:python-specific -macro _PY_PROTO_CMD_INTERNAL(File) { - .CMD=${cwd;rootdir;input:File} $GEN_PY_PROTOS --suffixes $PY_PROTO_SUFFIXES $PY_PROTO_MYPY_SUFFIX -- $_PY_PROTO_CMD_BASE($File __int___pb2.py $PY_PROTO_OPTS $PY_PROTO_OUTS_INTERNAL ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__int___pb2.py:File} ${nopath;noext;suf=_pb2.py:File}"} $PY_PROTO_MYPY_PLUGIN_INTERNAL) -} - -# tag:proto tag:java-specific -JAVA_PROTO_RUNTIME= - -# tag:proto tag:java-specific -### @usage: USE_JAVALITE() -### Use protobuf-javalite for Java -macro USE_JAVALITE() { - SET(JAVA_PROTO_RUNTIME javalite) -} - -# tag:proto tag:java-specific -JAVA_PROTO_RUNTIME_VERSION = 3.21.12 -JAVA_PROTO_COMMON_VERSION = 1.12.0 -KOTLIN_PROTO=no -KOTLIN_PROTO_PEERS= -when ($JAVA_PROTO_RUNTIME == "javalite") { - _JAVA_PROTO_LITE_ARG=lite: - JAVA_PROTOBUF_PEERS=contrib/java/com/google/protobuf/protobuf-javalite/${JAVA_PROTO_RUNTIME_VERSION} ${KOTLIN_PROTO_PEERS} - JAVA_GRPC_STUB=contrib/java/io/grpc/grpc-stub/1.43.2 - JAVA_GRPC_PROTOBUF=contrib/java/io/grpc/grpc-protobuf/1.43.2 -} -otherwise { - _JAVA_PROTO_LITE_ARG= - JAVA_PROTOBUF_PEERS=contrib/java/com/google/protobuf/protobuf-java/${JAVA_PROTO_RUNTIME_VERSION} ${KOTLIN_PROTO_PEERS} - JAVA_GRPC_STUB=contrib/java/io/grpc/grpc-stub/1.43.2 - JAVA_GRPC_PROTOBUF=contrib/java/io/grpc/grpc-protobuf/1.43.2 -} - -KOTLIN_PROTO_FLAGS= -# tag:proto tag:java-specific -macro _JAVA_PROTO_CMD(File) { - .CMD=${cwd;rootdir;input:File} $YMAKE_PYTHON ${input:"build/scripts/tared_protoc.py"} --tar-output ${output;norel;nopath;noext;suf=.jsrc:File} --protoc-out-dir $ARCADIA_BUILD_ROOT/java_out $PROTOC -I=./$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_ROOT --java_out=${_JAVA_PROTO_LITE_ARG}$ARCADIA_BUILD_ROOT/java_out ${KOTLIN_PROTO_FLAGS} $_PROTOC_FLAGS ${input;rootrel:File} ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} $JAVA_PROTO_ARGS - .SEM=proto_files ${input;rootrel:File} ${hide;output:File.jsrc} -} - - -# tag:proto tag:perl-specific tag:deprecated -### @usage: XS_PROTO(InputProto Dir Outputs...) # deprecated -### -### Generate Perl code from protobuf. -### In order to use this macro one should predict all outputs protoc will emit from input_proto file and enlist those as outputs. -macro XS_PROTO(File, Dir, Outputs...) { - .CMD=$PROTOC -I=${ARCADIA_ROOT}/${Dir} ${pre=-I=:_PROTO__INCLUDE} -I=${ARCADIA_ROOT} -I=${PROTOBUF_PATH} --perlxs_out=${BINDIR} $_PROTOC_FLAGS ${input:File} ${hide;output:Outputs} ${induced_deps=h+cpp;hide;nopath;noext;suf=.pb.h:File} - PEERDIR(${Dir}) - ADDINCL(${ARCADIA_BUILD_ROOT}/${Dir}) -} - -# tag:proto tag:fbs -### @usage: PROTO2FBS(InputProto) -### -### Produce flatbuf schema out of protobuf description. -macro PROTO2FBS(File) { - .CMD=${cwd:BINDIR} ${tool:"contrib/tools/flatc"} -I . -I ${ARCADIA_ROOT} --proto ${input:File} ${output;hide;norel;noext:File.fbs} ${kv;hide:"p FBS"} ${kv;hide:"pc yellow"} && $MOVE_FILE ${BINDIR}/${nopath;noext:File.fbs} ${output;norel;noext:File.fbs} -} - -# tag:proto tag:python-specific -when ($PY_PROTOS_FOR == "yes") { - PEERDIR+=contrib/libs/protobuf/python -} - -# tag:python-specific -macro _PY_EVLOG_CMD_BASE(File, Suf, Args...) { - .CMD=$_PY_PROTO_CMD_BASE($File $Suf $Args) - .PEERDIR=library/cpp/eventlog/proto -} - -# tag:python-specific tag:proto -macro _PY_EVLOG_CMD(File) { - .CMD=${cwd;rootdir;input:File} $_PY_EVLOG_CMD_BASE($File _ev_pb2.py) -} - -# tag:python-specific tag:proto -macro _PY_EVLOG_CMD_INTERNAL(File) { - .CMD=${cwd;rootdir;input:File} $GEN_PY_PROTOS --suffixes $PY_EVLOG_SUFFIXES -- $_PY_EVLOG_CMD_BASE($File __int___ev_pb2.py ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__int___ev_pb2.py:File} ${nopath;noext;suf=_ev_pb2.py:File}"}) -} - -# tag:java-specific tag:proto -macro _JAVA_EVLOG_CMD(File) { - .CMD=$COPY_CMD ${input:File} ${output;nopath;noext;norel;suf=_ev.proto:File} ${kv;hide:"p EV"} ${kv;hide:"pc yellow"} - .PEERDIR=library/cpp/eventlog/proto -} - # tag:sanitize RUN_NO_SANITIZE=$YMAKE_PYTHON ${input:"build/scripts/run_tool.py"} -- @@ -907,32 +407,6 @@ when ($ALLOCATOR == "LF") { # tag:sanitize SANITIZER_CFLAGS= -# tag:proto tag:grpc -_GRPC_ENABLED=no - -# tag:proto tag:grpc -### @usage: GRPC() -### -### Emit GRPC code for all .proto files in a PROTO_LIBRARY. -### This works for all available PROTO_LIBRARY versions (C++, Python 2.x, Python 3.x, Java and Go). -macro GRPC() { - ENABLE(_GRPC_ENABLED) - - # C++ - CPP_PROTO_PLUGIN2(grpc_cpp contrib/tools/protoc/plugins/grpc_cpp .grpc.pb.cc .grpc.pb.h DEPS contrib/libs/grpc) - - # Python - PY_PROTO_PLUGIN(grpc_py _pb2_grpc.py contrib/tools/protoc/plugins/grpc_python DEPS contrib/python/grpcio) - - # Java - JAVA_PROTO_PLUGIN(grpc_java contrib/tools/protoc/plugins/grpc_java DEPS $JAVA_GRPC_STUB $JAVA_GRPC_PROTOBUF) - SET_APPEND(JAVA_PROTOBUF_PEERS contrib/java/javax/annotation/javax.annotation-api/1.3.1) -} - -macro GO_PROTO_USE_V2() { - ENABLE(GO_PROTO_V2) -} - # tag:fbs FLATC_FLAGS_VALUE= ### @usage: FLATC_FLAGS(flags...) @@ -1004,8 +478,6 @@ macro _JAVA_FLATC_CMD(SRC) { } # variables must be defined for all module types to make generate_mf.py work -MODULE_PREFIX= -MODULE_SUFFIX= NEED_PLATFORM_PEERDIRS=yes PEERDIR_TEST_TOOL=yes @@ -2852,20 +2324,6 @@ module CI_GROUP: _BARE_UNIT { # tag:generic tag:internal TOUCH_UNIT_MF=$TOUCH_UNIT && $GENERATE_MF -# tag:proto -### @usage: RESOLVE_PROTO() -### -### Enable include resolving within UNIONs and let system .proto being resolved -### among .proto/.gztproto imports -### -### Note: it is currently impossible to enable resolving only for .proto, so resolving is enabled for all supported files -### also we only add ADDINCL for stock protobuf. So use this macro with care: it may cause resolving problems those are -### to be addressed by either ADDINCLs or marking them as TEXT. Please contact devtools for details. -macro RESOLVE_PROTO() { - SET(DONT_RESOLVE_INCLUDES no) - ADDINCL(FOR proto $PROTOBUF_PATH) -} - ### @usage: UNION(name) ### ### Collection of PEERDIR dependencies, files and artifacts. @@ -3199,31 +2657,6 @@ macro ADD_COMPILABLE_TRANSLIT(TranslitTable, NGrams, Name, Options...) { RUN_PROGRAM(dict/tools/make_ngrams -i ${NGrams} ${Options} ${__translitname_lower} IN ${NGrams} STDOUT ${__ngrams_table}) } -# tag:python-specific tag:proto tag:deprecated -### @usage GENERATE_PY_PROTOS(ProtoFiles...) # deprecated -### -### Generate python bindings for protobuf files. -### Macro is obsolete and not recommended for use! -macro GENERATE_PY_PROTOS(FILES...) { - foreach (FILE : $FILES) { - _PY_PROTO_CMD($FILE) - } -} - -# tag:python-specific tag:proto -macro _GENERATE_PY_PROTOS_INTERNAL(FILES...) { - foreach (FILE : $FILES) { - _PY_PROTO_CMD_INTERNAL($FILE) - } -} - -# tag:python-specific -macro _GENERATE_PY_EVS_INTERNAL(FILES...) { - foreach (FILE : $FILES) { - _PY_EVLOG_CMD_INTERNAL($FILE) - } -} - macro _COPY_FILE_IMPL(FILE, AUTO_DST="", NOAUTO_DST="", OUTPUT_INCLUDES[], INDUCED_DEPS[], OUTPUT_INCLUDES_INP[]) { .CMD=$COPY_CMD ${input:FILE} ${output:AUTO_DST} ${output;noauto:NOAUTO_DST} ${output_include;hide:OUTPUT_INCLUDES} ${output_include;from_input;hide:OUTPUT_INCLUDES_INP} $INDUCED_DEPS ${kv;hide:"p CP"} ${kv;hide:"pc light-cyan"} .SEM=copy_file ${input:FILE} ${output:AUTO_DST} ${output;noauto:NOAUTO_DST} @@ -5653,212 +5086,6 @@ macro _EXPOSE(Args...) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${output;suf=$_HASH_HELPER($Args).ya_exposed:"empty_"} ${input;hide:Args} } -### @usage: LIST_PROTO([TO list.proto] Files...) # deprecated -### -### Create list of .proto files in a list-file (should be .proto, files.proto by default) -### with original .proto-files as list's dependencies. -### -### This allows to process files listed, passing list as an argument to the processor -### -### TODO: proper implementation needed -macro LIST_PROTO(TO="files.proto", Files...) { - .CMD=$YMAKE_PYTHON3 ${input:"build/scripts/list.py"} ${Files} ${input;hide:Files} ${stdout;output;noauto:TO} ${output_include;from_input;hide:Files} && $YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${output;noext;suf=.pb.h:TO} - _EXPOSE(${TO}) -} - -# tag:proto -macro _PROTO_DESC_CMD(File) { - .CMD=${cwd;rootdir;input:File} $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_PATH --descriptor_set_out=${output;suf=.desc:File} --include_source_info $_PROTOC_FLAGS ${input;rootrel:File} -} - -_PROTO_DESC_MERGE_CMD=$YMAKE_PYTHON ${input:"build/scripts/merge_files.py"} $TARGET $AUTO_INPUT ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"} -_PROTO_DESC_MERGE_PEERS_CMD=$YMAKE_PYTHON ${input:"build/scripts/merge_files.py"} $TARGET $PEERS $SRCS_GLOBAL ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"} - -NEED_GOOGLE_PROTO_PEERDIRS=yes - -CPP_PROTO_LIBRARY_SEM=$CPP_LIBRARY_SEM - -JAVA_PROTO_LIBRARY_SEM=$BUILD_PROTO_JAR_SEM $_GRADLE_EXPORT_PUBLISHING_SEM - -# tag:proto -### @usage: PROTO_LIBRARY() -### -### Build some varian of protocol buffers library. -### -### The particular variant is selected based on where PEERDIR to PROTO_LIBRARY comes from. -### -### Now supported 5 variants: C++, Java, Python 2.x, Python 3.x and Go. -### When PEERDIR comes from module for particular language appropriate variant is selected. -### PROTO_LIBRARY also supports emission of GRPC code if GRPC() macro is specified. -### Notes: -### - Python versions emit C++ code in addition to Python as optimization. -### - In some PROTO_LIBRARY-es Java or Python versions are excluded via EXCLUDE_TAGS macros due to incompatibilities. -### - Use from DEPENDS or BUNDLE is not allowed -### -### Documentation: https://wiki.yandex-team.ru/yatool/proto_library/ -### -### See: [GRPC()](#macro_GRPC), [OPTIMIZE_PY_PROTOS()](#macro_OPTIMIZE_PY_PROTOS), [INCLUDE_TAGS()](#macro_INCLUDE_TAGS), [EXCLUDE_TAGS()](#macro_EXCLUDE_TAGS) -multimodule PROTO_LIBRARY { - module CPP_PROTO : LIBRARY { - .ALLOWED=_EXPOSE LIST_PROTO - # TODO(svidyuk): think about marker which forces semantics inheritance - .SEM=CPP_PROTO_LIBRARY_SEM - FORCE_COVERAGE_DISABLED=yes - ENABLE(CPP_PROTO) - ENABLE(GEN_PROTO) - NO_CLANG_TIDY() - SET(PEERDIR_TAGS CPP_PROTO) - - when ($BUILD_PROTO_AS_EVLOG == "yes" && $USE_VANILLA_PROTOC == "yes") { - _OK=no - } - ASSERT(_OK BUILD_PROTO_AS_EVLOG and USE_VANILLA_PROTOC are incompatible yet) - - MODULE_SUFFIX=$_CPP_PROTO_MODULE_SUFFIX - MODULE_PREFIX=$_CPP_PROTO_MODULE_PREFIX - - when ($_COMMON_GOOGLE_APIS != "None") { - PEERDIR += contrib/libs/googleapis-common-protos - } - } - - module JAVA_PROTO: EXTERNAL_JAVA_LIBRARY { - .EXTS=.jsrc - .ALLOWED=GRPC - .SEM=JAVA_PROTO_LIBRARY_SEM - SET(PEERDIR_TAGS JAVA_PROTO) - ENABLE(JAVA_PROTO) - PEERDIR+=$JAVA_PROTOBUF_PEERS - - when ($KOTLIN_PROTO == "yes") { - KOTLIN_PROTO_PEERS=contrib/java/com/google/protobuf/protobuf-kotlin/${JAVA_PROTO_RUNTIME_VERSION} - KOTLIN_PROTO_FLAGS=--kotlin_out=$ARCADIA_BUILD_ROOT/java_out - } - when ($GRADLE_EXPORT_PUBLISHING == "yes") { - _GRADLE_EXPORT_PUBLISHING_SEM=$_DO_GRADLE_EXPORT_PUBLISHING_SEM - } - - .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_SKIFF CPP_PROTO_PLUGIN2 PY_PROTO_PLUGIN YMAPS_SPROTO RESOURCE - ADDINCL(FOR proto $PROTOBUF_PATH) - - when ($_COMMON_GOOGLE_APIS != "None") { - PEERDIR += contrib/java/com/google/api/grpc/proto-google-common-protos/${JAVA_PROTO_COMMON_VERSION} - ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos - } - } - - module PY_PROTO: PY2_LIBRARY { - .ALIASES=SRCS=PY_SRCS - .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS - .PEERDIRSELF=CPP_PROTO - .SEM=IGNORED - SET(PEERDIR_TAGS PY2 PY_PROTO) - ENABLE(PY_PROTO) - OPTIMIZE_PY_PROTOS() - OBJ_SUF=.py2 - # Can not use NO_LINT(), because is not allowed outside of contrib directory - SET(_NO_LINT_VALUE none_internal) - - when ($_COMMON_GOOGLE_APIS != "None") { - PEERDIR += contrib/libs/googleapis-common-protos - } - - _IGNORE_SELF_PEERS= - _CPP_PROTO_LIBRARY=${MODDIR}/$_CPP_PROTO_MODULE_PREFIX$REALPRJNAME$_CPP_PROTO_MODULE_SUFFIX - when ($OPTIMIZE_PY_PROTOS_FLAG == "no") { - _IGNORE_PEERDIRSELF=CPP_PROTO - } - SET_APPEND(_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL $_CPP_PROTO_LIBRARY) - } - - module PY3_PROTO: PY3_LIBRARY { - .ALIASES=SRCS=PY_SRCS - .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS - .PEERDIRSELF=CPP_PROTO - .SEM=IGNORED - SET(PEERDIR_TAGS PY3 PY3_PROTO) - ENABLE(PY3_PROTO) - OPTIMIZE_PY_PROTOS() - OBJ_SUF=.py3 - # Can not use NO_LINT(), because is not allowed outside of contrib directory - SET(_NO_LINT_VALUE none_internal) - - when ($_COMMON_GOOGLE_APIS != "None") { - PEERDIR += contrib/libs/googleapis-common-protos - } - - _IGNORE_SELF_PEERS= - _CPP_PROTO_LIBRARY=${MODDIR}/$_CPP_PROTO_MODULE_PREFIX$REALPRJNAME$_CPP_PROTO_MODULE_SUFFIX - when ($OPTIMIZE_PY_PROTOS_FLAG == "no") { - _IGNORE_PEERDIRSELF=CPP_PROTO - } - SET_APPEND(_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL $_CPP_PROTO_LIBRARY) - } - - module GO_PROTO: GO_LIBRARY { - .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO - .SEM=IGNORED - SET(PEERDIR_TAGS GO GO_PROTO) - ENABLE(GO_PROTO) - - when ($_COMMON_GOOGLE_APIS == "None") { - } - elsewhen ($_COMMON_GOOGLE_APIS == "") { - PEERDIR += $_GO_COMMON_GOOGLE_APIS - ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos - } - otherwise { - PEERDIR += $_COMMON_GOOGLE_APIS - ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos - } - } - - module DESC_PROTO: _BARE_UNIT { - .CMD=_PROTO_DESC_MERGE_CMD - .SEM=IGNORED - .EXTS=.desc - .NODE_TYPE=Library - .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO RESOURCE GO_PROTO_PLUGIN GRPC - - SET(PEERDIR_TAGS DESC_PROTO) - ENABLE(DESC_PROTO) - MODULE_SUFFIX=.self.protodesc - SET(MODULE_TYPE LIBRARY) - - MACRO_ALIAS(EVLOG_CMD _PROTO_DESC_CMD) - MACRO_ALIAS(PROTO_CMD _PROTO_DESC_CMD) - - when ($_COMMON_GOOGLE_APIS != "None") { - PEERDIR += contrib/libs/googleapis-common-protos - } - - when ($NEED_GOOGLE_PROTO_PEERDIRS == "yes") { - when ($USE_VANILLA_PROTOC == "yes") { - PEERDIR += contrib/libs/protobuf_std/builtin_proto/protos_from_protobuf - } - otherwise { - PEERDIR += contrib/libs/protobuf/builtin_proto/protos_from_protoc - } - } - } -} - -module PROTO_DESCRIPTIONS: _BARE_UNIT { - .CMD=_PROTO_DESC_MERGE_PEERS_CMD - .PEERDIR_POLICY=as_build_from - .NODE_TYPE=Library - .RESTRICTED=SRCS - .FINAL_TARGET=yes - - SET(PEERDIR_TAGS DESC_PROTO) - SET(MODULE_SUFFIX .protodesc) - SET(MODULE_TYPE PROTO_DESCRIPTIONS) -} - -module PROTO_REGISTRY: PROTO_DESCRIPTIONS { - SET(MODULE_TYPE PROTO_REGISTRY) -} - # tag:fbs _FBS_NAMESPACE_MAP_GLOBAL= |