Merge pull request #11788 from ydb-platform/mergelibs-241120-1113

Library import 241120-1113

1 files changed, 242 insertions, 148 deletions

diff --git a/build/conf/proto.conf b/build/conf/proto.conf
index a55bfbe7bd..17704bdfd0 100644
--- a/build/conf/proto.conf
+++ b/build/conf/proto.conf
@@ -655,6 +655,162 @@ CPP_PROTO_LIBRARY_SEM=$CPP_LIBRARY_SEM
 
 JAVA_PROTO_LIBRARY_SEM=$BUILD_PROTO_JAR_SEM $_GRADLE_EXPORT_PUBLISHING_SEM
 
+module _CPP_PROTO : LIBRARY {
+    .ALLOWED=_EXPOSE LIST_PROTO
+    # TODO(svidyuk): think about marker which forces semantics inheritance
+    .SEM=CPP_PROTO_LIBRARY_SEM
+    FORCE_COVERAGE_DISABLED=yes
+    ENABLE(CPP_PROTO)
+    ENABLE(GEN_PROTO)
+    NO_CLANG_TIDY()
+    CPP_PROTOLIBS_DEBUG_INFO()
+    SET(PEERDIR_TAGS)
+
+    when ($BUILD_PROTO_AS_EVLOG == "yes" && $USE_VANILLA_PROTOC == "yes") {
+        _OK=no
+    }
+    ASSERT(_OK BUILD_PROTO_AS_EVLOG and USE_VANILLA_PROTOC are incompatible yet)
+
+    MODULE_SUFFIX=$_CPP_PROTO_MODULE_SUFFIX
+    MODULE_PREFIX=$_CPP_PROTO_MODULE_PREFIX
+
+    when ($_COMMON_GOOGLE_APIS != "None") {
+        PEERDIR += contrib/libs/googleapis-common-protos
+    }
+}
+
+module _JAVA_PROTO: EXTERNAL_JAVA_LIBRARY {
+    .EXTS=.jsrc
+    .ALLOWED=GRPC
+    .SEM=JAVA_PROTO_LIBRARY_SEM
+    SET(PEERDIR_TAGS JAVA)
+    ENABLE(JAVA_PROTO)
+    DISABLE(_NEED_SBOM_INFO)
+    PEERDIR+=$JAVA_PROTOBUF_PEERS
+
+    when ($KOTLIN_PROTO == "yes") {
+        KOTLIN_PROTO_PEERS=contrib/java/com/google/protobuf/protobuf-kotlin/${JAVA_PROTO_RUNTIME_VERSION}
+        KOTLIN_PROTO_FLAGS=--kotlin_out=$ARCADIA_BUILD_ROOT/java_out
+    }
+    when ($GRADLE_EXPORT_PUBLISHING == "yes") {
+        _GRADLE_EXPORT_PUBLISHING_SEM=$_DO_GRADLE_EXPORT_PUBLISHING_SEM
+    }
+
+    .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_SKIFF CPP_PROTO_PLUGIN2 PY_PROTO_PLUGIN YMAPS_SPROTO RESOURCE
+    ADDINCL(FOR proto $PROTOBUF_INCLUDE_PATH)
+
+    when ($_COMMON_GOOGLE_APIS != "None") {
+        PEERDIR += contrib/java/com/google/api/grpc/proto-google-common-protos/${JAVA_PROTO_COMMON_VERSION}
+        ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos
+    }
+}
+
+module _PY_PROTO: PY2_LIBRARY {
+    .ALIASES=SRCS=PY_SRCS
+    .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS
+    .SEM=IGNORED
+    SET(PEERDIR_TAGS PY2)
+    ENABLE(PY_PROTO)
+    DISABLE(_NEED_SBOM_INFO)
+    OPTIMIZE_PY_PROTOS()
+    OBJ_SUF=.py2
+    # Can not use NO_LINT(), because is not allowed outside of contrib directory
+    SET(_NO_LINT_VALUE none_internal)
+
+    when ($_COMMON_GOOGLE_APIS != "None") {
+        PEERDIR += contrib/libs/googleapis-common-protos
+    }
+
+    _IGNORE_SELF_PEERS=
+    _CPP_PROTO_LIBRARY=${MODDIR}/$_CPP_PROTO_MODULE_PREFIX$REALPRJNAME$_CPP_PROTO_MODULE_SUFFIX
+    when ($OPTIMIZE_PY_PROTOS_FLAG == "no") {
+        _IGNORE_PEERDIRSELF=CPP_PROTO
+    }
+    SET_APPEND(_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL $_CPP_PROTO_LIBRARY)
+}
+
+module _PY3_PROTO: PY3_LIBRARY {
+    .ALIASES=SRCS=PY_SRCS
+    .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS
+    .SEM=IGNORED
+    SET(PEERDIR_TAGS PY3)
+    ENABLE(PY3_PROTO)
+    DISABLE(_NEED_SBOM_INFO)
+    OPTIMIZE_PY_PROTOS()
+    OBJ_SUF=.py3
+    # Can not use NO_LINT(), because is not allowed outside of contrib directory
+    SET(_NO_LINT_VALUE none_internal)
+
+    when ($_COMMON_GOOGLE_APIS != "None") {
+        PEERDIR += contrib/libs/googleapis-common-protos
+    }
+
+    _IGNORE_SELF_PEERS=
+    _CPP_PROTO_LIBRARY=${MODDIR}/$_CPP_PROTO_MODULE_PREFIX$REALPRJNAME$_CPP_PROTO_MODULE_SUFFIX
+    when ($OPTIMIZE_PY_PROTOS_FLAG == "no") {
+        _IGNORE_PEERDIRSELF=CPP_PROTO
+    }
+    SET_APPEND(_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL $_CPP_PROTO_LIBRARY)
+}
+
+module _GO_PROTO: GO_LIBRARY {
+    .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO
+    .SEM=IGNORED
+    SET(PEERDIR_TAGS GO)
+    ENABLE(GO_PROTO)
+
+    when ($_COMMON_GOOGLE_APIS == "None") {
+    }
+    elsewhen ($_COMMON_GOOGLE_APIS == "") {
+        PEERDIR += $_GO_COMMON_GOOGLE_APIS
+        ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos
+    }
+    otherwise {
+        PEERDIR += $_COMMON_GOOGLE_APIS
+        ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos
+    }
+}
+
+module _TS_PROTO: _TS_PROTO_IMPL {
+    DISABLE(_NEED_SBOM_INFO)
+}
+
+module _TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
+    .IGNORED=PEERDIR
+    DISABLE(_NEED_SBOM_INFO)
+}
+
+module _DESC_PROTO: _BARE_UNIT {
+    .CMD=_PROTO_DESC_MERGE_CMD
+    .SEM=IGNORED
+    .EXTS=.desc .rawproto
+    .NODE_TYPE=Library
+    .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO RESOURCE GO_PROTO_PLUGIN GRPC
+    .ALIASES=SRCS=_SRCS_NO_GLOBAL
+
+    ENABLE(DESC_PROTO)
+    DISABLE(_NEED_SBOM_INFO)
+    MODULE_SUFFIX=.self.protodesc
+    SET(MODULE_TYPE LIBRARY)
+    SET(_MODDIR_HASH ${hash:MODDIR})
+
+    _EVLOG_CMDLINE=$_PROTO_DESC_CMDLINE
+    _PROTO_CMDLINE=$_PROTO_DESC_CMDLINE
+
+    when ($_COMMON_GOOGLE_APIS != "None") {
+        PEERDIR += contrib/libs/googleapis-common-protos
+    }
+
+    when ($NEED_GOOGLE_PROTO_PEERDIRS == "yes") {
+        when ($USE_VANILLA_PROTOC == "yes") {
+            PEERDIR += contrib/libs/protobuf_std/builtin_proto/protos_from_protobuf
+        }
+        otherwise {
+            PEERDIR += contrib/libs/protobuf/builtin_proto/protos_from_protoc
+        }
+    }
+}
+
 # tag:proto
 ### @usage: PROTO_LIBRARY()
 ###
@@ -674,173 +830,44 @@ JAVA_PROTO_LIBRARY_SEM=$BUILD_PROTO_JAR_SEM $_GRADLE_EXPORT_PUBLISHING_SEM
 ###
 ### See: [GRPC()](#macro_GRPC), [OPTIMIZE_PY_PROTOS()](#macro_OPTIMIZE_PY_PROTOS), [INCLUDE_TAGS()](#macro_INCLUDE_TAGS), [EXCLUDE_TAGS()](#macro_EXCLUDE_TAGS)
 multimodule PROTO_LIBRARY {
-    module CPP_PROTO : LIBRARY {
-        .ALLOWED=_EXPOSE LIST_PROTO
-        # TODO(svidyuk): think about marker which forces semantics inheritance
-        .SEM=CPP_PROTO_LIBRARY_SEM
-        FORCE_COVERAGE_DISABLED=yes
-        ENABLE(CPP_PROTO)
-        ENABLE(GEN_PROTO)
-        NO_CLANG_TIDY()
-        CPP_PROTOLIBS_DEBUG_INFO()
-        SET(PEERDIR_TAGS CPP_PROTO)
-
-        when ($BUILD_PROTO_AS_EVLOG == "yes" && $USE_VANILLA_PROTOC == "yes") {
-            _OK=no
-        }
-        ASSERT(_OK BUILD_PROTO_AS_EVLOG and USE_VANILLA_PROTOC are incompatible yet)
-
-        MODULE_SUFFIX=$_CPP_PROTO_MODULE_SUFFIX
-        MODULE_PREFIX=$_CPP_PROTO_MODULE_PREFIX
-
-        when ($_COMMON_GOOGLE_APIS != "None") {
-            PEERDIR += contrib/libs/googleapis-common-protos
-        }
+    module CPP_PROTO: _CPP_PROTO {
+        SET_APPEND(PEERDIR_TAGS CPP_PROTO)
     }
 
-    module JAVA_PROTO: EXTERNAL_JAVA_LIBRARY {
-        .EXTS=.jsrc
-        .ALLOWED=GRPC
-        .SEM=JAVA_PROTO_LIBRARY_SEM
-        SET(PEERDIR_TAGS JAVA JAVA_PROTO)
-        ENABLE(JAVA_PROTO)
-        DISABLE(_NEED_SBOM_INFO)
-        PEERDIR+=$JAVA_PROTOBUF_PEERS
-
-        when ($KOTLIN_PROTO == "yes") {
-            KOTLIN_PROTO_PEERS=contrib/java/com/google/protobuf/protobuf-kotlin/${JAVA_PROTO_RUNTIME_VERSION}
-            KOTLIN_PROTO_FLAGS=--kotlin_out=$ARCADIA_BUILD_ROOT/java_out
-        }
-        when ($GRADLE_EXPORT_PUBLISHING == "yes") {
-            _GRADLE_EXPORT_PUBLISHING_SEM=$_DO_GRADLE_EXPORT_PUBLISHING_SEM
-        }
-
-        .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_SKIFF CPP_PROTO_PLUGIN2 PY_PROTO_PLUGIN YMAPS_SPROTO RESOURCE
-        ADDINCL(FOR proto $PROTOBUF_INCLUDE_PATH)
-
-        when ($_COMMON_GOOGLE_APIS != "None") {
-            PEERDIR += contrib/java/com/google/api/grpc/proto-google-common-protos/${JAVA_PROTO_COMMON_VERSION}
-            ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos
-        }
+    module JAVA_PROTO: _JAVA_PROTO {
+        SET_APPEND(PEERDIR_TAGS JAVA_PROTO)
     }
 
-    module PY_PROTO: PY2_LIBRARY {
-        .ALIASES=SRCS=PY_SRCS
-        .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS
+    module PY_PROTO: _PY_PROTO {
+        SET_APPEND(PEERDIR_TAGS PY_PROTO)
         .PEERDIRSELF=CPP_PROTO
-        .SEM=IGNORED
-        SET(PEERDIR_TAGS PY2 PY_PROTO)
-        ENABLE(PY_PROTO)
-        DISABLE(_NEED_SBOM_INFO)
-        OPTIMIZE_PY_PROTOS()
-        OBJ_SUF=.py2
-        # Can not use NO_LINT(), because is not allowed outside of contrib directory
-        SET(_NO_LINT_VALUE none_internal)
-
-        when ($_COMMON_GOOGLE_APIS != "None") {
-            PEERDIR += contrib/libs/googleapis-common-protos
-        }
-
-        _IGNORE_SELF_PEERS=
-        _CPP_PROTO_LIBRARY=${MODDIR}/$_CPP_PROTO_MODULE_PREFIX$REALPRJNAME$_CPP_PROTO_MODULE_SUFFIX
-        when ($OPTIMIZE_PY_PROTOS_FLAG == "no") {
-            _IGNORE_PEERDIRSELF=CPP_PROTO
-        }
-        SET_APPEND(_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL $_CPP_PROTO_LIBRARY)
     }
 
-    module PY3_PROTO: PY3_LIBRARY {
-        .ALIASES=SRCS=PY_SRCS
-        .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS
+    module PY3_PROTO: _PY3_PROTO {
+        SET_APPEND(PEERDIR_TAGS PY3_PROTO)
         .PEERDIRSELF=CPP_PROTO
-        .SEM=IGNORED
-        SET(PEERDIR_TAGS PY3 PY3_PROTO)
-        ENABLE(PY3_PROTO)
-        DISABLE(_NEED_SBOM_INFO)
-        OPTIMIZE_PY_PROTOS()
-        OBJ_SUF=.py3
-        # Can not use NO_LINT(), because is not allowed outside of contrib directory
-        SET(_NO_LINT_VALUE none_internal)
-
-        when ($_COMMON_GOOGLE_APIS != "None") {
-            PEERDIR += contrib/libs/googleapis-common-protos
-        }
-
-        _IGNORE_SELF_PEERS=
-        _CPP_PROTO_LIBRARY=${MODDIR}/$_CPP_PROTO_MODULE_PREFIX$REALPRJNAME$_CPP_PROTO_MODULE_SUFFIX
-        when ($OPTIMIZE_PY_PROTOS_FLAG == "no") {
-            _IGNORE_PEERDIRSELF=CPP_PROTO
-        }
-        SET_APPEND(_WHOLE_ARCHIVE_LIBS_VALUE_GLOBAL $_CPP_PROTO_LIBRARY)
     }
 
-    module GO_PROTO: GO_LIBRARY {
-        .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO
-        .SEM=IGNORED
-        SET(PEERDIR_TAGS GO GO_PROTO)
-        ENABLE(GO_PROTO)
-
-        when ($_COMMON_GOOGLE_APIS == "None") {
-        }
-        elsewhen ($_COMMON_GOOGLE_APIS == "") {
-            PEERDIR += $_GO_COMMON_GOOGLE_APIS
-            ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos
-        }
-        otherwise {
-            PEERDIR += $_COMMON_GOOGLE_APIS
-            ADDINCL += GLOBAL FOR proto ${ARCADIA_ROOT}/contrib/libs/googleapis-common-protos
-        }
+    module GO_PROTO: _GO_PROTO {
+        SET_APPEND(PEERDIR_TAGS GO_PROTO)
     }
 
-    module TS_PROTO: _TS_PROTO_IMPL {
+    module TS_PROTO: _TS_PROTO {
         # opt-in. We don't want to have TS_PROTO by default
         # To include TS_PROTO user have to set INCLUDE_TAGS(TS_PROTO TS_PREPARE_DEPS) in ya.make
         .INCLUDE_TAG=no
-        .EPILOGUE=_TS_CONFIG_EPILOGUE
         .PEERDIRSELF=TS_PREPARE_DEPS
-        DISABLE(_NEED_SBOM_INFO)
-        
-        when ($_COMMON_GOOGLE_APIS != "None") {
-            PEERDIR += contrib/libs/googleapis-common-protos
-        }
+        .EPILOGUE=_TS_CONFIG_EPILOGUE
+        SET_APPEND(PEERDIR_TAGS TS_PROTO)
     }
 
-    module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
+    module TS_PREPARE_DEPS: _TS_PREPARE_DEPS {
         .INCLUDE_TAG=no
-        .IGNORED=PEERDIR SRCS GENERATE_ENUM_SERIALIZATION
-        DISABLE(_NEED_SBOM_INFO)
-    }
-
-    module DESC_PROTO: _BARE_UNIT {
-        .CMD=_PROTO_DESC_MERGE_CMD
-        .SEM=IGNORED
-        .EXTS=.desc .rawproto
-        .NODE_TYPE=Library
-        .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO RESOURCE GO_PROTO_PLUGIN GRPC
-        .ALIASES=SRCS=_SRCS_NO_GLOBAL
-
-        SET(PEERDIR_TAGS DESC_PROTO)
-        ENABLE(DESC_PROTO)
-        DISABLE(_NEED_SBOM_INFO)
-        MODULE_SUFFIX=.self.protodesc
-        SET(MODULE_TYPE LIBRARY)
-        SET(_MODDIR_HASH ${hash:MODDIR})
-
-        _EVLOG_CMDLINE=$_PROTO_DESC_CMDLINE
-        _PROTO_CMDLINE=$_PROTO_DESC_CMDLINE
-
-        when ($_COMMON_GOOGLE_APIS != "None") {
-            PEERDIR += contrib/libs/googleapis-common-protos
-        }
+        SET_APPEND(PEERDIR_TAGS TS_PREPARE_DEPS)
+    }
 
-        when ($NEED_GOOGLE_PROTO_PEERDIRS == "yes") {
-            when ($USE_VANILLA_PROTOC == "yes") {
-                PEERDIR += contrib/libs/protobuf_std/builtin_proto/protos_from_protobuf
-            }
-            otherwise {
-                PEERDIR += contrib/libs/protobuf/builtin_proto/protos_from_protoc
-            }
-        }
+    module DESC_PROTO: _DESC_PROTO {
+        SET_APPEND(PEERDIR_TAGS DESC_PROTO)
     }
 }
 
@@ -852,7 +879,7 @@ module PROTO_DESCRIPTIONS: _BARE_UNIT {
     .FINAL_TARGET=yes
 
     SET(MODULE_TAG PROTO_DESCRIPTIONS)
-    SET(PEERDIR_TAGS DESC_PROTO)
+    SET(PEERDIR_TAGS DESC_PROTO DESC_PROTO_FROM_SCHEMA)
     SET(MODULE_SUFFIX .protodesc)
     SET(MODULE_TYPE PROTO_DESCRIPTIONS)
 }
@@ -862,6 +889,73 @@ module PROTO_REGISTRY: PROTO_DESCRIPTIONS {
     SET(MODULE_TYPE PROTO_REGISTRY)
 }
 
+# tag:proto
+### @usage: PROTO_SCHEMA()
+###
+### Build some variant of protocol buffers library or proto descriptions.
+###
+### When used as a PEERDIR from a language module like GO_PROGRAM it behaves like PROTO_LIBRARY.
+### When built directly it produces proto descriptions. When required by RECURSE it produces
+### both proto descriptions and language libraries.
+### PROTO_SCHEMA can depend on PROTO_LIBRARY, but PROTO_LIBRARY cannot depend on PROTO_SCHEMA.
+###
+### See: [PROTO_LIBRARY()](#module_PROTO_LIBRARY)
+multimodule PROTO_SCHEMA {
+    module CPP_PROTO_FROM_SCHEMA: _CPP_PROTO {
+        DISABLE(START_TARGET)
+        SET_APPEND(PEERDIR_TAGS CPP_PROTO CPP_PROTO_FROM_SCHEMA)
+    }
+
+    module JAVA_PROTO_FROM_SCHEMA: _JAVA_PROTO {
+        DISABLE(START_TARGET)
+        SET_APPEND(PEERDIR_TAGS JAVA_PROTO JAVA_PROTO_FROM_SCHEMA)
+    }
+
+    module PY_PROTO_FROM_SCHEMA: _PY_PROTO {
+        .PEERDIRSELF=CPP_PROTO_FROM_SCHEMA
+        DISABLE(START_TARGET)
+        SET_APPEND(PEERDIR_TAGS PY_PROTO PY_PROTO_FROM_SCHEMA)
+    }
+
+    module PY3_PROTO_FROM_SCHEMA: _PY3_PROTO {
+        .PEERDIRSELF=CPP_PROTO_FROM_SCHEMA
+        DISABLE(START_TARGET)
+        SET_APPEND(PEERDIR_TAGS PY3_PROTO PY3_PROTO_FROM_SCHEMA)
+    }
+
+    module GO_PROTO_FROM_SCHEMA: _GO_PROTO {
+        DISABLE(START_TARGET)
+        SET_APPEND(PEERDIR_TAGS GO_PROTO GO_PROTO_FROM_SCHEMA)
+    }
+
+    module TS_PROTO_FROM_SCHEMA: _TS_PROTO {
+        # opt-in. We don't want to have TS_PROTO by default
+        # To include TS_PROTO user have to set INCLUDE_TAGS(TS_PROTO TS_PREPARE_DEPS) in ya.make
+        .INCLUDE_TAG=no
+        .PEERDIRSELF=TS_PREPARE_DEPS_FROM_SCHEMA
+        .EPILOGUE=_TS_CONFIG_EPILOGUE
+        DISABLE(START_TARGET)
+        SET_APPEND(PEERDIR_TAGS TS_PROTO TS_PROTO_FROM_SCHEMA)
+    }
+
+    module TS_PREPARE_DEPS_FROM_SCHEMA: _TS_PREPARE_DEPS {
+        .INCLUDE_TAG=no
+        DISABLE(START_TARGET)
+        SET_APPEND(PEERDIR_TAGS TS_PREPARE_DEPS TS_PREPARE_DEPS_FROM_SCHEMA)
+    }
+
+    module DESC_PROTO_FROM_SCHEMA: _DESC_PROTO {
+        DISABLE(START_TARGET)
+        SET_APPEND(PEERDIR_TAGS DESC_PROTO DESC_PROTO_FROM_SCHEMA)
+    }
+
+    module PROTO_DESCRIPTIONS_: PROTO_DESCRIPTIONS {
+        .PEERDIRSELF=DESC_PROTO_FROM_SCHEMA
+        .FINAL_TARGET=yes
+        SET_APPEND(PEERDIR_TAGS DESC_PROTO_FROM_SCHEMA)
+    }
+}
+
 macro EVLOG_CMD(SRC) {
     .CMD=$_EVLOG_CMDLINE
     .SEM=$_EVLOG_CMDLINE