Add DECLARE_IN_DIRS macro

aaac7e22ec66cd83845962458e8aefb7a9fa58dd

2 files changed, 107 insertions, 0 deletions

diff --git a/build/plugins/macros_with_error.py b/build/plugins/macros_with_error.py
index 81cd709678..003f4a8bf3 100644
--- a/build/plugins/macros_with_error.py
+++ b/build/plugins/macros_with_error.py
@@ -30,3 +30,31 @@ def onassert(unit, *args):
     if val and val.lower() == "no":
         msg = ' '.join(args[1:])
         ymake.report_configure_error(msg)
+
+
+def onvalidate_in_dirs(unit, *args):
+    files_var = args[0]
+    pattern = args[1]
+    no_srcdir = args[2] == '--'
+    srcdir = '' if no_srcdir else args[2]
+    dirs = args[3:] if no_srcdir else args[4:]
+    pfx = '[[imp]]DECLARE_IN_DIRS[[rst]]: '
+
+    if '**' in pattern:
+        ymake.report_configure_error(f"{pfx}'**' in files mask is prohibited. Seen [[unimp]]{pattern}[[rst]]")
+        unit.set([files_var, ""])
+
+    for pat in ('*', '?'):
+        if pat in srcdir:
+            ymake.report_configure_error(
+                f"{pfx}'{pat}' in [[imp]]SRCDIR[[rst]] argument is prohibited. Seen [[unimp]]{srcdir}[[rst]]"
+            )
+            unit.set([files_var, ""])
+
+    for dir in dirs:
+        for pat in ('*', '?', '$', '..'):
+            if pat in dir:
+                ymake.report_configure_error(
+                    f"{pfx}'{pat}' in [[imp]]DIRS[[rst]] argument is prohibited. Seen [[unimp]]{dir}[[rst]]"
+                )
+                unit.set([files_var, ""])
diff --git a/build/ymake.core.conf b/build/ymake.core.conf
index 3f3a69dbc0..7db071847b 100644
--- a/build/ymake.core.conf
+++ b/build/ymake.core.conf
@@ -4386,6 +4386,85 @@ macro PREPARE_INDUCED_DEPS(VAR, For, Deps...) {
     SET($VAR \$_FMT_INDUCED_DEPS($For $Deps))
 }
 
+
+
+### Helper macro for DECLARE_IN_DIRS to add proper '/' to non-empty SRCDIR and form proper EXCLUDES
+macro _DECL_IN_DIR_GLOB(var_prefix, pattern, rec_part, SRCDIR="", EXCLBASE="", EXCLUDES[], DIRS[], DUMMY[]) {
+    _PATTERN=${pre=${SRCDIR};suf=/${rec_part}${pattern}:DIRS}
+    _EXCLUDES=${EXCLBASE}**/ya.make ${EXCLBASE}**/a.yaml ${pre=${EXCLBASE}:EXCLUDES}
+
+    _LATE_GLOB(_${var_prefix}_FILES ${_PATTERN} EXCLUDE ${_EXCLUDES})
+    SET(${var_prefix}_PATTERN ${_PATTERN})
+    SET(${var_prefix}_EXCLUDES ${_EXCLUDES})
+}
+
+### @usage: DECLARE_IN_DIR(var_prefix files_mask DIRS dirs [RECURSIVE] [EXCLUDES excludes] [SRCDIR srcdir])
+###
+### This macro allow passing content of directories to macros like `RUN_PROGRAM` and `RUN_PYTHON3` as IN parameter.
+###
+### The content is matched by following rules:
+### - The files are looked in <srcdir>. The srcdir is relative to module directory and defaulted to module directory.
+### - Inside <srcdir> files are looked in all <dirs>, recursively or not depending on RECURSIVE parameter.
+### - Files are matched by file_mask which may contain * or ?.
+### - <excludes> are then applied over matched files. Excludes are regular globs including recursive parts support.
+###
+### Taking `var_prefix` macro declared 4 variables:
+### - <var_prefix>_FILES - the file list matched by the macro using rules above. This variable can be passed to `IN` parameter of `RUN_PROGRAM` and alikes.
+###                        Also it may be passed escaped as argument to tool/script. See example below.
+### - <var_prefix>_PATTERNS - the glob patterns used for match.
+### - <var_prefix>_EXCLUDES - exclude patterns from EXCLUDES argument and ones to exclude ya.make and a.yaml
+### - <var_prefix>_SRCDIR - value of SRCDIR argument
+###
+### Parameters:
+### - var_prefix - Mandatory prefix of variables the macro declares
+### - file_mask - Mandatory glob-like mask for files
+###   file_mask should not conatain '**'
+### - DIRS dirs - Mandatory list of dirs relative to srcdir or current one in which files should be looked
+###   Dirs cannot contain ${ARCADIA_ROOT} (and other similar vars), '..', '*' or '?'.
+### - RECURSIVE - Optional request to lookup dirs recursively. Default is non-recursive lookup
+### - EXCLUDES excludes - Optional list of globs to exclude from match.
+### - SRCDIR srcdir - Optional directory (relative to current one) to apply globs. Default is the current dir.
+###   We strongly discourage this, but srcdir may contain '..' or start from ${ARCADIA_ROOT} for root-relative addressing.
+###   scrdir cannot contain any of '*' or '?'
+###
+### Examples:
+### ```
+### DECLARE_IN_DIRS(TXT *.txt DIRS . EXCLUDE .*.txt **/.*.txt)
+### # file list requires escaping as argument
+### RUN_PYTHON3(concat.py \${TXT_FILES} IN ${TXT_FILES} STDOUT concatenated.txt)
+### ```
+###
+### ```
+### DECLARE_IN_DIRS(ALL_TXT *.txt SRCDIR txt RECURSIVE DIRS design rules EXCLUDE all.txt)
+### RUN_PYTHON3(concat_all.py --dirs ${MODDIR}/${ALL_TXT_SRCDIR} --pattern ${ALL_TXT_PATTERNS} --exclude ${ALL_TXT_EXCLUDES} IN ${ALL_TXT_FILES} STDOUT concatenated.txt)
+### ```
+###
+### ```
+### DECLARE_IN_DIRS(D_TXT *.txt SRCDIR txt/design EXCLUDE all.txt DIRS .)
+### DECLARE_IN_DIRS(R_TXT *.txt SRCDIR txt/rules DIRS .)
+### RUN_PYTHON3(concat_all.py --dirs ${MODDIR}/${D_TXT_SRCDIR} ${MODDIR}/${R_TXT_SRCDIR} --patterns ${D_TXT_PATTERNS} ${R_TXT_PATTERNS} --exclude ${D_TXT_EXCLUDES} IN ${D_TXT_FILES} ${R_TXT_FILES} STDOUT concatenated.txt)
+### ```
+###
+### Notes:
+### 1. All 'ya.make' and 'a.yaml' files are excluded from match.
+### 2. Matched files are never parsed for dependencies even though they shall be passed to IN, not to IN_NOPARSE.
+### 3. The list of files expanded late and will not work in macros like SRCS. This macro only meant for use with generating macros like RUN_PROGRAM processing entire matching list with one command.
+### 4. We support extended file mask syntax for multiple masks like "(*.cpp|*.h)". However, this will be preserved in <var_prefix>_PATTERNS variable and so tool/script either should support such syntax or
+###    or should not rely on value of the variable for actual matching.
+### 5. There is known issue with empty match and escaped substitution like `concat.py \${TXT_FILES}`. It may result in weird errors and can be workarounded by extra argument like `concat.py - \${TXT_FILES}`
+### 6. EXCLUDES work differently with SRCDIR is specified. Use discriminating tail of SRCDIR in order to match exact files non-recursively.
+###    E.g. if SRCDIR is a/b/zz and EXCLUDE is *.x the exclude will work recursively on all matches including zz's child dierctories. To limit match to zz's level use EXCLUDE zz/*.x instead.
+### 6. Parameters of macro are somewhat validated and we may add extra checks in the fulture including protection over too broad match.
+###
+macro DECLARE_IN_DIRS(var_prefix, PATTERN, SRCDIR="", RECURSIVE?"**/":"", EXCLUDES[], DIRS[]) {
+    _DECL_IN_DIR_GLOB($var_prefix $PATTERN $RECURSIVE ${pre=SRCDIR ;suf=/:SRCDIR} ${pre=EXCLBASE ${ARCADIA_ROOT}/**/ DUMMY :SRCDIR} EXCLUDES ${EXCLUDES} DIRS ${DIRS}) 
+
+    SET(${var_prefix}_FILES \$_${var_prefix}_FILES)
+    SET(${var_prefix}_SRCDIR $SRCDIR)
+    VALIDATE_IN_DIRS(${var_prefix}_FILES $PATTERN $SRCDIR -- $DIRS)
+}
+
+
 ### @usage: RUN_PROGRAM(tool_path args... [CWD dir] [ENV key=value...] [TOOL tools...] [IN[_NOPARSE] inputs...] [OUT[_NOAUTO] outputs...] [STDOUT[_NOAUTO] output] [OUTPUT_INCLUDES output_includes...] [INDUCED_DEPS $VARs...])
 ###
 ### Run a program from arcadia.