add ymake export to ydb

1 files changed, 402 insertions, 0 deletions

diff --git a/build/conf/docs.conf b/build/conf/docs.conf
new file mode 100644
index 0000000000..8fe8ed3486
--- /dev/null
+++ b/build/conf/docs.conf
@@ -0,0 +1,402 @@
+#
+# Documentation build support
+#
+
+# tag:docs
+TOUCH_DOCS=$YMAKE_PYTHON3 ${input:"build/scripts/touch.py"} ${kv;hide:"p DC"} ${kv;hide:"pc light-cyan"} $TARGET
+TOUCH_DOCS_MF=$TOUCH_DOCS && $GENERATE_MF
+
+# tag:docs
+### @usage: DOCS_COPY_FILES(FROM src_dir [NAMESPCE dst_dir] files...)
+###
+### Copy files from src_dir to $BINDIR/dst_dir
+macro DOCS_COPY_FILES(FROM="${CURDIR}", NAMESPACE=".", FILES...) {
+    .CMD=$YMAKE_PYTHON3 ${input:"build/scripts/copy_docs_files.py"} ${input;hide:"build/scripts/process_command_files.py"} --source-root $ARCADIA_ROOT --build-root $ARCADIA_BUILD_ROOT --src-dir $FROM --dst-dir $BINDIR/$NAMESPACE $FILES ${input;hide;context=TEXT;pre=${FROM}/:FILES} ${output;hide;pre=${NAMESPACE}/:FILES}
+}
+
+# tag:docs
+_DOCS_USE_PLANTUML=no
+_DOCS_EXTRA_TOOLS=
+_DOCS_EXTRA_INPUTS=
+_DOCS_ENV=
+_DOCS_KV=${kv;hide:"p DO"} ${kv;hide:"pc light-cyan"} ${kv;hide:"show_out yes"}
+
+_DOCS_PLANTUML_ENV=\
+${env:"JAVA_PATH=$JDK19_RESOURCE_GLOBAL/bin/java"} \
+${env:"PLANTUML_PATH=contrib/tools/plantuml/plantuml.run.cp.jar"} \
+${env:"_JAVA_OPTIONS='-Dsun.awt.fontconfig=contrib/java/openjdk-fontconfig/fontconfig.properties -Djava.awt.headless=true'"} \
+${env:"LANG=en_US.UTF-8"} \
+${env:"LC_ALL=C.UTF-8"}
+
+_DOCS_SRCS_VALUE=
+_DOCS_VARS_FLAG=
+
+_DOCS_YFM_OUTPUT_FORMAT=
+_DOCS_YFM_BOOK_OUTPUT_FORMAT=--output-format html --allowHTML
+_DOCS_YFM_LIB_OUTPUT_FORMAT=--output-format md --add-map-file --allow-custom-resources
+
+_DOCS_YFM_CMDLINE=\
+${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON3 ${input:"build/scripts/extract_docs.py"} ${input;hide:"build/scripts/process_command_files.py"} --skip-prefix $ARCADIA_BUILD_ROOT --dest-dir $BINDIR/__s ${rootrel:PEERS} \
+&& ${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON3 ${input:"build/scripts/stdout2stderr.py"} $YFM_TOOL_RESOURCE_GLOBAL/yfm-docs --input $BINDIR/__s --output $BINDIR/__docsbuild $_DOCS_VARS_FLAG $_DOCS_YFM_OUTPUT_FORMAT --config ${input:CONFIG} $_DOCS_EXTRA_TOOLS ${hide;input:EXTRA_INPUTS} $_DOCS_ENV \
+&& $YMAKE_PYTHON3 ${input:"build/scripts/tar_sources.py"} --output $TARGET --input $BINDIR/__docsbuild $_DOCS_KV
+
+# tag:internal tag:docs
+macro _DOCS_YFM_CMD_IMPL(CONFIG, EXTRA_INPUTS[]) {
+    .CMD=$_DOCS_YFM_CMDLINE
+}
+
+_DOCS_YFM_CMD=$_DOCS_YFM_CMD_IMPL($_DOCS_CONFIG_VALUE EXTRA_INPUTS $_DOCS_EXTRA_INPUTS)
+
+# tag:docs
+### This module is intended for internal use only. Common parts for DOCS and MKDOCS multimodules
+### should be defined here.
+module _DOCS_BARE_UNIT: _BARE_UNIT {
+    .ALLOWED=DOCS_DIR DOCS_CONFIG DOCS_VARS
+    .CMD=TOUCH_DOCS_MF
+    .FINAL_TARGET=no
+    .NODE_TYPE=Bundle
+    .PEERDIR_POLICY=as_include
+
+    ENABLE(_DOCS_BARE_UNIT)
+
+    SET(MODULE_SUFFIX .tar.gz)
+    SET(MODULE_LANG DOCS)
+}
+
+# tag:docs tag:internal
+_DOCS_DIR_INTERNAL_NAMESPACE=
+
+# tag:internal tag:docs
+_DOCS_LIBRARY_CMDLINE=\
+$YMAKE_PYTHON3 ${input:"build/scripts/copy_docs_files_to_dir.py"} ${input;hide:"build/scripts/process_command_files.py"} $_DOCS_SRCS_VALUE $_DOCS_DIR_VALUE $_DOCS_BIN_DIR_VALUE --dest-dir $BINDIR/__s --source-root $ARCADIA_ROOT --build-root $ARCADIA_BUILD_ROOT ${input;context=TEXT:INCLUDE_SRCS} \
+&& $YMAKE_PYTHON3 ${input:"build/scripts/tar_sources.py"} --output $TARGET --input $BINDIR/__s $_DOCS_KV
+
+# tag:internal tag:docs
+macro _DOCS_LIBRARY_CMD_IMPL(INCLUDE_SRCS[], EXTRA_INPUTS[]) {
+    .CMD=$_DOCS_LIBRARY_CMDLINE
+}
+
+_DOCS_LIBRARY_CMD=$_DOCS_LIBRARY_CMD_IMPL(INCLUDE_SRCS $_DOCS_INCLUDE_SOURCES_VALUE)
+
+# tag:docs
+module DOCS_LIBRARY: _DOCS_BARE_UNIT {
+    .CMD=_DOCS_LIBRARY_CMD
+    .ALIASES=SRCS=_DOCS_SRCS DOCS_DIR=_YFM_DOCS_DIR
+    .EPILOGUE=_DOCS_LIBRARY_EPILOGUE
+    .NODE_TYPE=Library
+
+    ENABLE(DOCS_LIBRARY)
+
+    SET(MODULE_TYPE LIBRARY)
+    SET(MODULE_TAG DOCS_LIBRARY)
+    SET(PEERDIR_TAGS DOCS_LIBRARY)
+
+    SET(MODULE_SUFFIX .docslib)
+}
+
+# tag:docs
+macro _DOCS_SRCS(SRCDIR=".", EXCLUDE[], INCLUDE...) {
+    SET(_VAR_DOCS_SRCS_SALT __SRCDIR__ $SRCDIR __EXCLUDE__ $EXCLUDE __INCLUDE__ $INCLUDE)
+    SET(_DOCS_SRCS_GLOB uniq_docs_${hash:_VAR_DOCS_SRCS_SALT})
+    _GLOB(${_DOCS_SRCS_GLOB} ${pre=${SRCDIR}/:INCLUDE} EXCLUDE ${EXCLUDE})
+    SET_APPEND(_DOCS_INCLUDE_SOURCES_VALUE ${input:$_DOCS_SRCS_GLOB})
+    SET(_YFM_DOCS_DIR_DEFAULT_VALUE __dummy_dir__)
+}
+
+# tag:docs
+macro _DOCS_LIBRARY_EPILOGUE() {
+    _YFM_DOCS_DIR($_YFM_DOCS_DIR_DEFAULT_VALUE)
+    _SET_DOCS_BIN_DIR_FLAG($_DOCS_DIR_INTERNAL_NAMESPACE $MODDIR)
+}
+
+# tag:docs
+### This module is intended for internal use only. Common parts for submodules of DOCS multimodule
+### should be defined here.
+module _DOCS_BASE_UNIT: _DOCS_BARE_UNIT {
+    .ALIASES=DOCS_DIR=_YFM_DOCS_DIR
+    .NODE_TYPE=Library
+
+    ENABLE(_DOCS_BASE_UNIT)
+
+    PEERDIR+=build/platform/yfm
+    DOCS_CONFIG($_DOCS_YFM_DEFAULT_CONFIG)
+}
+
+# tag:internal tag:docs
+### _DOCS_YFM_USE_PLANTUML() # internal
+###
+### This macr sets appropriate dependencies for use of plantuml plugin
+macro _DOCS_YFM_USE_PLANTUML() {
+    when ($DOCSLIB == "yes") {
+        PEERDIR+=build/platform/java/jdk/jdk19 contrib/java/openjdk-fontconfig
+
+        _DOCS_ENV+=$_DOCS_PLANTUML_ENV
+        _DOCS_EXTRA_TOOLS+=${hide;tool:"contrib/tools/plantuml"}
+    }
+}
+
+# tag:docs
+### @usage: DOCS()
+###
+### Documentation project multimodule.
+###
+### When built directly, via RECURSE, DEPENDS or BUNDLE the output artifact is docs.tar.gz with statically generated site.
+### When PEERDIRed from other DOCS() module behaves like a UNION (supplying own content and dependencies to build target).
+### Peerdirs from modules other than DOCS are not accepted.
+### Most usual macros are not accepted, only used with the macros DOCS_DIR(), DOCS_CONFIG(), DOCS_VARS().
+###
+### @see: [DOCS_DIR()](#macro_DOCS_DIR), [DOCS_CONFIG()](#macro_DOCS_CONFIG), [DOCS_VARS()](#macro_DOCS_VARS).
+multimodule DOCS {
+    module DOCSBOOK: _DOCS_BASE_UNIT {
+        .CMD=_DOCS_YFM_CMD
+        .FINAL_TARGET=yes
+        .PEERDIR_POLICY=as_build_from
+        .IGNORED=DOCS_DIR DOCS_INCLUDE_SOURCES DOCS_COPY_FILES PEERDIR PYTHON RUN_PROGRAM RUN_PYTHON3 RUN_LUA RUN_JAVA_PROGRAM FROM_SANDBOX SRCS
+        .PEERDIRSELF=DOCSLIB
+
+        ENABLE(DOCSBOOK)
+
+        SET(MODULE_TAG DOCSBOOK)
+        SET(PEERDIR_TAGS DOCSLIB)
+
+        _DOCS_YFM_OUTPUT_FORMAT=$_DOCS_YFM_BOOK_OUTPUT_FORMAT
+
+        PROCESS_DOCS()
+    }
+
+    module DOCSLIB: _DOCS_BASE_UNIT {
+        .CMD=_DOCS_YFM_CMD
+        .PEERDIR_POLICY=as_build_from
+        .IGNORED=DOCS_DIR DOCS_INCLUDE_SOURCES DOCS_COPY_FILES PEERDIR PYTHON RUN_PROGRAM RUN_PYTHON3 RUN_LUA RUN_JAVA_PROGRAM FROM_SANDBOX SRCS
+        .PEERDIRSELF=DOCSLIB_INTERNAL
+        .PROXY=yes
+
+        ENABLE(DOCSLIB)
+
+        SET(MODULE_TAG DOCSLIB)
+        SET(PEERDIR_TAGS DOCSLIB_INTERNAL)
+
+        REALPRJNAME=preprocessed
+
+        _DOCS_YFM_OUTPUT_FORMAT=$_DOCS_YFM_LIB_OUTPUT_FORMAT
+
+        PROCESS_DOCS()
+    }
+
+    module DOCSLIB_INTERNAL: DOCS_LIBRARY {
+        .IGNORED=DOCS_CONFIG
+        .EPILOGUE=_DOCS_LIBRARY_EPILOGUE
+
+        ENABLE(DOCSLIB_INTERNAL)
+        DISABLE(START_TARGET)
+
+        SET(MODULE_TAG DOCSLIB_INTERNAL)
+        SET(PEERDIR_TAGS DOCSLIB_EXTERNAL DOCS_LIBRARY)
+        # additional .fake extension make this output suppressed by ya-bin
+        SET(MODULE_SUFFIX .docslib.fake)
+
+        SET(_DOCS_DIR_INTERNAL_NAMESPACE .)
+
+        REALPRJNAME=__docs_internal
+    }
+
+    module DOCSLIB_EXTERNAL: DOCS_LIBRARY {
+        .IGNORED=DOCS_CONFIG
+        .EPILOGUE=_DOCS_LIBRARY_EPILOGUE
+
+        ENABLE(DOCSLIB_EXTERNAL)
+        DISABLE(START_TARGET)
+
+        SET(MODULE_TAG DOCSLIB_EXTERNAL)
+        SET(PEERDIR_TAGS DOCSLIB_EXTERNAL DOCS_LIBRARY)
+
+        REALPRJNAME=__docs_external
+    }
+}
+
+_DOCS_FS_TOOLS=$YMAKE_PYTHON3 ${input:"build/scripts/fs_tools.py"} ${input;hide:"build/scripts/process_command_files.py"}
+
+_DOCS_MKDOCS_CMDLINE_SUFFIX=
+_DOCS_MKDOCS_BOOK_CMDLINE_SUFFIX=${pre=--dep ;ext=preprocessed.tar.gz:PEERS}
+_DOCS_MKDOCS_LIB_CMDLINE_SUFFIX=--preprocess-md-only
+
+_DOCS_MKDOCS_CMDLINE=\
+${cwd:ARCADIA_ROOT} $_DOCS_FS_TOOLS copy_all_files $_MKDOCS_DOCS_DIR_VALUE $BINDIR/__s $_DOCS_SRCS_VALUE \
+&& $YMAKE_PYTHON3 ${input:"build/scripts/copy_files_to_dir.py"} ${input;hide:"build/scripts/process_command_files.py"} --dest-dir $BINDIR/__s --skip-prefix $ARCADIA_ROOT --skip-prefix $ARCADIA_BUILD_ROOT ${input;context=TEXT:INCLUDE_SRCS} \
+&& ${cwd:BINDIR} $YMAKE_PYTHON3 ${input:"build/scripts/mkdocs_builder_wrapper.py"} $ARCADIA_BUILD_ROOT ${tool:"tools/mkdocs_builder"} --docs-dir $BINDIR/__s --output-tar $TARGET --config ${input:CONFIG} $_DOCS_VARS_FLAG $_DOCS_MKDOCS_CMDLINE_SUFFIX $_DOCS_EXTRA_TOOLS ${hide;input:EXTRA_INPUTS} $_DOCS_ENV $_DOCS_KV
+
+# tag:internal tag:docs
+macro _DOCS_MKDOCS_CMD_IMPL(CONFIG, INCLUDE_SRCS[], EXTRA_INPUTS[]) {
+    .CMD=$_DOCS_MKDOCS_CMDLINE
+}
+
+_DOCS_MKDOCS_CMD=$_DOCS_MKDOCS_CMD_IMPL($_DOCS_CONFIG_VALUE INCLUDE_SRCS $_DOCS_INCLUDE_SOURCES_VALUE)
+
+_DOCS_YFM_DEFAULT_CONFIG=$MODDIR/.yfm
+_DOCS_MKDOCS_DEFAULT_CONFIG=$MODDIR/mkdocs.yml
+
+# tag:docs
+### This module is intended for internal use only. Common parts for submodules of MKDOCS multimodule
+### should be defined here.
+module _MKDOCS_BASE_UNIT: _DOCS_BARE_UNIT {
+    .ALIASES=DOCS_DIR=_MKDOCS_DOCS_DIR
+
+    ENABLE(_MKDOCS_BASE_UNIT)
+
+    DOCS_CONFIG($_DOCS_MKDOCS_DEFAULT_CONFIG)
+}
+
+# tag:internal tag:docs
+### _MKDOCS_EPILOOGUE() # internal
+###
+### This macro executes macros which should be envoked after all user
+### specified macros in the ya.make file
+macro _MKDOCS_EPILOGUE() {
+    _LATE_GLOB(_DOCS_SRCS_GLOB ${pre=${ARCADIA_ROOT}/;suf=/**/*:_MKDOCS_DOCS_DIR_VALUE})
+    SET(_DOCS_SRCS_VALUE \${input;hide:_DOCS_SRCS_GLOB})
+}
+
+# tag:docs
+### @usage: MKDOCS()
+###
+### Documentation project multimodule.
+###
+### When built directly, via RECURSE, DEPENDS or BUNDLE the output artifact is docs.tar.gz with statically generated site (using mkdocs as builder).
+### When PEERDIRed from other MKDOCS() module behaves like a UNION (supplying own content and dependencies to build target).
+### Peerdirs from modules other than MKDOCS are not accepted.
+### Most usual macros are not accepted, only used with the macros DOCS_DIR(), DOCS_CONFIG(), DOCS_VARS().
+###
+### @see: [DOCS_DIR()](#macro_DOCS_DIR), [DOCS_CONFIG()](#macro_DOCS_CONFIG), [DOCS_VARS()](#macro_DOCS_VARS).
+multimodule MKDOCS {
+    module MKDOCSBOOK: _MKDOCS_BASE_UNIT {
+        .CMD=_DOCS_MKDOCS_CMD
+        .EPILOGUE=_MKDOCS_EPILOGUE
+        .FINAL_TARGET=yes
+        .PEERDIR_POLICY=as_build_from
+
+        ENABLE(MKDOCSBOOK)
+
+        SET(MODULE_TYPE PROGRAM)
+        SET(PEERDIR_TAGS MKDOCSLIB)
+        SET(MODULE_TAG MKDOCSBOOK)
+
+        _DOCS_MKDOCS_CMDLINE_SUFFIX=$_DOCS_MKDOCS_BOOK_CMDLINE_SUFFIX
+
+        PROCESS_MKDOCS()
+    }
+
+    module MKDOCSLIB: _MKDOCS_BASE_UNIT {
+        .CMD=_DOCS_MKDOCS_CMD
+        .EPILOGUE=_MKDOCS_EPILOGUE
+        .PEERDIR_POLICY=as_include
+
+        ENABLE(MKDOCSLIB)
+
+        SET(MODULE_TYPE LIBRARY)
+        SET(PEERDIR_TAGS MKDOCSLIB)
+        SET(MODULE_TAG MKDOCSLIB)
+
+        REALPRJNAME=preprocessed
+
+        _DOCS_MKDOCS_CMDLINE_SUFFIX=$_DOCS_MKDOCS_LIB_CMDLINE_SUFFIX
+
+        PROCESS_MKDOCS()
+    }
+}
+
+# tag:docs
+_DOCS_USE_PLANTUML=
+### @usage: USE_PLANTUML()
+###
+### Use PlantUML plug-in for yfm builder to render UML diagrams into documentation
+macro USE_PLANTUML() {
+    ENABLE(_DOCS_USE_PLANTUML)
+}
+
+# tag:docs
+_DOCS_DIR_VALUE=
+_DOCS_BIN_DIR_VALUE=
+### @usage: DOCS_DIR(path)
+###
+### Specify directory with source .md files for DOCS multimodule if it differs from project directory.
+### Path must be Arcadia root relative.
+###
+### @see: [DOCS](#multimodule_DOCS)
+macro DOCS_DIR(Dir) {
+    ENABLE(UNUSED_MACRO)
+}
+
+# tag:docs tag:internal
+macro _APPEND_DOCS_DIR_FLAG(DIR, NAMESPACE, DYMMY...) {
+    SET_APPEND(_DOCS_DIR_VALUE --docs-dir $DIR $NAMESPACE)
+}
+
+# tag:docs tag:internal
+macro _SET_DOCS_BIN_DIR_FLAG(NAMESPACE, DUMMY...) {
+    SET(_DOCS_BIN_DIR_VALUE --bin-dir $BINDIR $NAMESPACE $AUTO_INPUT)
+}
+
+# tag:docs tag:internal
+_YFM_DOCS_DIR_DEFAULT_VALUE=$MODDIR
+_YFM_DEFAULT_NAMESPACE=
+### @usage: DOCS_DIR(path) # internal
+macro _YFM_DOCS_DIR(DIR) {
+    _APPEND_DOCS_DIR_FLAG($DIR $_DOCS_DIR_INTERNAL_NAMESPACE $DIR)
+
+    SET(_VAR_DOCS_DIR_SALT __MODDIR__ $MODDIR __DIR__ $DIR)
+    SET(_DOCS_DIR_GLOB uniq_docs_dir_${hash:_VAR_DOCS_DIR_SALT})
+    _LATE_GLOB(${_DOCS_DIR_GLOB} ${ARCADIA_ROOT}/$DIR/**/*)
+    SET_APPEND(_DOCS_SRCS_VALUE \${input;hide:$_DOCS_DIR_GLOB})
+
+    # We set the value of var _YFM_DOCS_DIR_DEFAULT_VALUE to some non-existing dir. This value
+    # will be used in _DOCS_LIBRARY_EPILOGUE calls. In case when this macro _YFM_DOCS_DIR is
+    # explicitly called in DOCS_LIBRARY module $MODDIR as default DOCS_DIR for DOCS_LIBRARY will
+    # be ignore.
+    SET(_YFM_DOCS_DIR_DEFAULT_VALUE __dummy_dir__)
+}
+
+# tag:docs tag:internal
+_MKDOCS_DOCS_DIR_VALUE=$MODDIR
+### @usage: DOCS_DIR(path) # internal
+macro _MKDOCS_DOCS_DIR(Dir) {
+    SET(_MKDOCS_DOCS_DIR_VALUE $Dir)
+}
+
+# tag:docs
+_DOCS_DEFAULT_CONFIG=
+_DOCS_CONFIG_VALUE=$_DOCS_DEFAULT_CONFIG
+### @usage: DOCS_CONFIG(path)
+###
+### Specify path to config file for DOCS multimodule if it differs from default path.
+### If used for [MKDOCS](#multimodule_MKDOCS) multimodule the default path is "%%project_directory%%/mkdocs.yml".
+### If used for [DOCS](#multimodule_DOCS) multimodule the default path is "%%project_directory%%/.yfm".
+### Path must be either Arcadia root relative.
+###
+### @see: [DOCS](#multimodule_DOCS)
+macro DOCS_CONFIG(File) {
+    SET(_DOCS_CONFIG_VALUE $File)
+}
+
+# tag:docs
+_DOCS_VARS_VALUE=
+### @usage: DOCS_VARS(variable1=value1 variable2=value2 ...)
+###
+### Specify a set of default values of template variables for DOCS multimodule.
+### There must be no spaces around "=". Values will be treated as strings.
+###
+### @see: [DOCS](#multimodule_DOCS)
+macro DOCS_VARS(Args...) {
+    SET_APPEND(_DOCS_VARS_VALUE $Args)
+}
+
+# tag:docs
+_DOCS_INCLUDE_SOURCES_VALUE=
+### @usage: DOCS_INCLUDE_SOURCES(path...)
+###
+### Specify a list of paths to source code files which will be used as text includes in a documentation project.
+### Paths must be Arcadia root relative.
+###
+### @see: [DOCS](#multimodule_DOCS)
+macro DOCS_INCLUDE_SOURCES(Args...) {
+    SET_APPEND(_DOCS_INCLUDE_SOURCES_VALUE $Args)
+}