aboutsummaryrefslogtreecommitdiffstats
path: root/devtools/ya
diff options
context:
space:
mode:
authorrobot-piglet <robot-piglet@yandex-team.com>2023-12-02 01:45:21 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2023-12-02 02:42:50 +0300
commit9c43d58f75cf086b744cf4fe2ae180e8f37e4a0c (patch)
tree9f88a486917d371d099cd712efd91b4c122d209d /devtools/ya
parent32fb6dda1feb24f9ab69ece5df0cb9ec238ca5e6 (diff)
downloadydb-9c43d58f75cf086b744cf4fe2ae180e8f37e4a0c.tar.gz
Intermediate changes
Diffstat (limited to 'devtools/ya')
-rw-r--r--devtools/ya/chameleon_bin/__main__.py59
-rw-r--r--devtools/ya/chameleon_bin/recipe.inc32
-rw-r--r--devtools/ya/chameleon_bin/ya.make12
-rw-r--r--devtools/ya/handlers/dump/__init__.py1161
-rw-r--r--devtools/ya/handlers/dump/gen_conf_docs.py384
-rw-r--r--devtools/ya/handlers/dump/ya.make52
-rw-r--r--devtools/ya/handlers/gc/__init__.py342
-rw-r--r--devtools/ya/handlers/gc/ya.make37
-rw-r--r--devtools/ya/handlers/gen_config/__init__.py48
-rw-r--r--devtools/ya/handlers/gen_config/gen_config.py282
-rw-r--r--devtools/ya/handlers/gen_config/ya.make23
-rw-r--r--devtools/ya/handlers/ide/__init__.py734
-rw-r--r--devtools/ya/handlers/ide/ya.make31
-rw-r--r--devtools/ya/handlers/java/__init__.py49
-rw-r--r--devtools/ya/handlers/java/helpers.py250
-rw-r--r--devtools/ya/handlers/java/ya.make24
-rw-r--r--devtools/ya/handlers/make/__init__.py32
-rw-r--r--devtools/ya/handlers/make/ya.make26
-rw-r--r--devtools/ya/handlers/maven_import/__init__.py31
-rw-r--r--devtools/ya/handlers/maven_import/ya.make28
-rw-r--r--devtools/ya/handlers/package/__init__.py88
-rw-r--r--devtools/ya/handlers/package/opts/__init__.py691
-rw-r--r--devtools/ya/handlers/package/opts/ya.make19
-rw-r--r--devtools/ya/handlers/package/ya.make27
-rw-r--r--devtools/ya/handlers/test/__init__.py40
-rw-r--r--devtools/ya/handlers/test/ya.make21
-rw-r--r--devtools/ya/handlers/tool/__init__.py281
-rw-r--r--devtools/ya/handlers/tool/ya.make33
-rw-r--r--devtools/ya/handlers/ya.make151
-rw-r--r--devtools/ya/opensource/ya.conf92
30 files changed, 5080 insertions, 0 deletions
diff --git a/devtools/ya/chameleon_bin/__main__.py b/devtools/ya/chameleon_bin/__main__.py
new file mode 100644
index 0000000000..f79b940fdb
--- /dev/null
+++ b/devtools/ya/chameleon_bin/__main__.py
@@ -0,0 +1,59 @@
+import logging
+import os
+import shutil
+from library.python.testing.recipe import declare_recipe
+import yatest.common
+
+
+class ChameleonRecipe:
+ def __init__(self):
+ self.bin2_root: str = None
+ self.bin3_root: str = None
+ self.bin2: str = None
+ self.bin3: str = None
+
+ def post_init(self):
+ self.bin2_root = yatest.common.build_path("devtools/ya/bin")
+ self.bin3_root = yatest.common.build_path("devtools/ya/bin3")
+ self.bin2 = os.path.join(self.bin2_root, "ya-bin")
+ self.bin3 = os.path.join(self.bin3_root, "ya-bin")
+
+ logging.info("ya-bin2 exists: %s", os.path.exists(self.bin2))
+ logging.info("ya-bin3 exists: %s", os.path.exists(self.bin3))
+
+ @classmethod
+ def check_argv_py3(cls, argv):
+ return argv[0] == "yes"
+
+ def start(self, argv):
+ self.post_init()
+
+ logging.info("args: %s", argv)
+
+ assert not (os.path.exists(self.bin2) and os.path.exists(self.bin3)), "Only one version of ya-bin can be in DEPENDS"
+
+ if self.check_argv_py3(argv):
+ if not os.path.exists(self.bin2):
+ if not os.path.exists(self.bin2_root):
+ os.mkdir(self.bin2_root)
+ logging.debug("Create bin2 root folder: %s", self.bin2_root)
+ else:
+ logging.debug("Maybe test was restarted")
+ shutil.copy(self.bin3, self.bin2)
+ logging.debug("copy ya-bin3 to bin2 root: %s -> %s", self.bin3, self.bin2)
+
+
+ def stop(self, argv):
+ self.post_init()
+
+ logging.info("args: %s", argv)
+
+ if self.check_argv_py3(argv):
+ if os.path.exists(self.bin2):
+ logging.debug("Remove bin2 executable: %s", self.bin2)
+ # TOFIX v-korovin
+ os.remove(self.bin2)
+
+if __name__ == "__main__":
+ recipe = ChameleonRecipe()
+ declare_recipe(recipe.start, recipe.stop)
diff --git a/devtools/ya/chameleon_bin/recipe.inc b/devtools/ya/chameleon_bin/recipe.inc
new file mode 100644
index 0000000000..4e705cc996
--- /dev/null
+++ b/devtools/ya/chameleon_bin/recipe.inc
@@ -0,0 +1,32 @@
+DEPENDS(devtools/ya/chameleon_bin)
+
+INCLUDE(${ARCADIA_ROOT}/devtools/ya/opensource.inc)
+
+IF (PYTHON3)
+ DEPENDS(devtools/ya/bin3)
+ELSE()
+ IF (NOT OPENSOURCE)
+ DEPENDS(devtools/ya/bin)
+ ENDIF()
+ENDIF()
+
+IF (NEBIUS)
+ DATA(
+ arcadia/nebius/devtools/ya-bin/ya.conf
+ )
+ELSEIF (OPENSOURCE)
+ DATA(
+ arcadia/devtools/ya/opensource/ya.conf
+ )
+ELSE()
+ DATA(
+ arcadia/ya.conf
+ )
+ENDIF()
+
+DATA(
+ arcadia/ya
+)
+
+USE_RECIPE(devtools/ya/chameleon_bin/chameleon_bin ${PYTHON3})
+
diff --git a/devtools/ya/chameleon_bin/ya.make b/devtools/ya/chameleon_bin/ya.make
new file mode 100644
index 0000000000..3b418c2651
--- /dev/null
+++ b/devtools/ya/chameleon_bin/ya.make
@@ -0,0 +1,12 @@
+PY3_PROGRAM()
+
+PY_SRCS(__main__.py)
+
+PEERDIR(
+ library/python/testing/recipe
+ library/python/testing/yatest_common
+)
+
+NO_LINT()
+
+END()
diff --git a/devtools/ya/handlers/dump/__init__.py b/devtools/ya/handlers/dump/__init__.py
new file mode 100644
index 0000000000..77eb2f45d2
--- /dev/null
+++ b/devtools/ya/handlers/dump/__init__.py
@@ -0,0 +1,1161 @@
+from __future__ import absolute_import
+from __future__ import print_function
+import csv
+import pickle
+import exts.yjson as json
+import logging
+import os
+import sys
+import enum
+import six
+
+from build.build_facade import (
+ gen_all_loops,
+ gen_dir_loops,
+ gen_graph,
+ gen_licenses_list,
+ gen_forced_deps_list,
+ gen_modules,
+ gen_module_info,
+ gen_plan,
+ gen_json_graph,
+ gen_uids,
+ gen_srcdeps,
+ gen_dependencies,
+ gen_test_dart,
+ gen_conf,
+ gen_filelist,
+ gen_relation,
+)
+from handlers.dump.gen_conf_docs import dump_mmm_docs
+from build.dir_graph import reachable, gen_dir_graph
+from build.compilation_database import dump_compilation_database, COMPILATION_DATABASE_OPTS
+from exts.strtobool import strtobool
+from exts.tmp import temp_dir
+from core.yarg import (
+ CompositeHandler,
+ OptsHandler,
+ ArgConsumer,
+ FreeArgConsumer,
+ SetValueHook,
+ Options,
+ SetConstValueHook,
+ SetConstAppendHook,
+ UpdateValueHook,
+ SetAppendHook,
+ ExtendHook,
+ ArgsValidatingException,
+ Group,
+)
+from yalibrary.vcs import vcsversion
+from core.imprint import imprint
+from build.build_opts import (
+ YMakeDebugOptions,
+ YMakeBinOptions,
+ FlagsOptions,
+ CustomFetcherOptions,
+ ToolsOptions,
+ SandboxAuthOptions,
+ JavaSpecificOptions,
+)
+from build.build_opts import (
+ BuildTypeOptions,
+ BuildTargetsOptions,
+ ShowHelpOptions,
+ CustomBuildRootOptions,
+ ContinueOnFailOptions,
+)
+from build.build_opts import YMakeRetryOptions, ConfigurationPresetsOptions
+from core.common_opts import CrossCompilationOptions, YaBin3Options
+from test.explore import generate_tests_by_dart
+from devtools.ya.test.dartfile import decode_recipe_cmdline
+
+import app
+import app_config
+
+logger = logging.getLogger(__name__)
+
+
+class DumpYaHandler(CompositeHandler):
+ common_opts = [ShowHelpOptions()]
+
+ @staticmethod
+ def common_build_facade_opts(with_free_targets=True):
+ return [
+ FlagsOptions(),
+ CustomFetcherOptions(),
+ SandboxAuthOptions(),
+ YMakeBinOptions(),
+ YMakeRetryOptions(),
+ CrossCompilationOptions(),
+ BuildTypeOptions('release'),
+ ContinueOnFailOptions(),
+ BuildTargetsOptions(with_free=with_free_targets),
+ ToolsOptions(),
+ ConfigurationPresetsOptions(),
+ YaBin3Options(),
+ ]
+
+ def __init__(self):
+ CompositeHandler.__init__(self, description='Repository related information')
+ self['modules'] = OptsHandler(
+ action=app.execute(action=do_modules),
+ description='All modules',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ PeerDirectoriesOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ ],
+ )
+ self['module-info'] = OptsHandler(
+ action=app.execute(action=do_module_info),
+ description='Modules info',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ PeerDirectoriesOptions(),
+ DataOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ DumpModuleInfoOptions(),
+ ],
+ )
+ self['src-deps'] = OptsHandler(
+ action=app.execute(action=do_dump_srcdeps),
+ description='Dump of all source dependencies',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DumpSrcDepsOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ ],
+ )
+ self['dir-graph'] = OptsHandler(
+ action=app.execute(action=do_dir_graph),
+ description='Dependencies between directories',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DumpDirOptions(),
+ SplitByTypeOptions(),
+ DumpTestListOptions(),
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ LegacyDepsOptions(),
+ ],
+ )
+ self['dep-graph'] = OptsHandler(
+ action=app.execute(action=do_dep_graph),
+ description='Dependency internal graph',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DepGraphOutputOptions(),
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ LegacyDepsOptions(),
+ ],
+ )
+ self['dot-graph'] = OptsHandler(
+ action=app.execute(action=do_dot_graph),
+ description='Dependency between directories in dot format',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ PeerDirectoriesOptions(),
+ CustomBuildRootOptions(),
+ ],
+ )
+ self['json-dep-graph'] = OptsHandler(
+ action=app.execute(action=do_json_dep_graph),
+ description='Dependency graph as json',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ LegacyDepsOptions(),
+ ],
+ )
+ self['build-plan'] = OptsHandler(
+ action=app.execute(action=do_build_plan),
+ description='Build plan',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DepTraverseOptions(),
+ CustomBuildRootOptions(),
+ YMakeDebugOptions(),
+ ],
+ )
+ self['compile-commands'] = OptsHandler(
+ action=app.execute(action=dump_compilation_database),
+ description='JSON compilation database',
+ opts=COMPILATION_DATABASE_OPTS + [ToolsOptions(), CustomFetcherOptions(), SandboxAuthOptions()],
+ )
+ self['compilation-database'] = OptsHandler(
+ action=app.execute(action=dump_compilation_database),
+ description='Alias for compile-commands',
+ opts=COMPILATION_DATABASE_OPTS + [ToolsOptions(), CustomFetcherOptions(), SandboxAuthOptions()],
+ )
+ self['relation'] = OptsHandler(
+ action=app.execute(action=do_relation),
+ description='PEERDIR relations. Please don\'t run from the arcadia root.',
+ opts=self.common_opts
+ + self.common_build_facade_opts(False)
+ + [
+ RelationSrcDstOptions(free_dst=True),
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ PeerDirectoriesOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ ],
+ )
+ self['all-relations'] = OptsHandler(
+ action=app.execute(action=do_all_relations),
+ description='All relations between internal graph nodes in dot format. Please don\'t run from the arcadia root.',
+ opts=self.common_opts
+ + self.common_build_facade_opts(False)
+ + [
+ RelationSrcDstOptions(free_dst=True),
+ DumpAllRelationsOptions(),
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ PeerDirectoriesOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ ],
+ )
+ self['files'] = OptsHandler(
+ action=app.execute(action=do_files),
+ description='File list',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ FilesFilterOptions(),
+ DepTraverseOptions(),
+ DepFilterOptions(),
+ DataOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ ],
+ )
+ self['loops'] = OptsHandler(
+ action=app.execute(action=do_loops),
+ description='All loops in arcadia',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DepTraverseOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ ],
+ )
+ self['peerdir-loops'] = OptsHandler(
+ action=app.execute(action=do_peerdir_loops),
+ description='Loops by peerdirs',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ DepTraverseOptions(),
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ ],
+ )
+ self['imprint'] = OptsHandler(
+ action=app.execute(action=do_imprint),
+ description='Directory imprint',
+ opts=self.common_opts + [CustomFetcherOptions(), SandboxAuthOptions(), ToolsOptions()],
+ )
+ self['uids'] = OptsHandler(
+ action=app.execute(action=do_uids),
+ description='All targets uids',
+ opts=self.common_opts + self.common_build_facade_opts(),
+ visible=False,
+ )
+ self['test-list'] = OptsHandler(
+ action=app.execute(action=do_test_list),
+ description='All test entries',
+ opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions()],
+ )
+ self['json-test-list'] = OptsHandler(
+ action=app.execute(action=do_json_test_list),
+ description='All test entries as json',
+ opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions()],
+ )
+ self['conf'] = OptsHandler(
+ action=app.execute(action=do_conf),
+ description='Print build conf',
+ opts=self.common_opts
+ + [
+ FlagsOptions(),
+ CustomFetcherOptions(),
+ SandboxAuthOptions(),
+ CrossCompilationOptions(),
+ ToolsOptions(),
+ BuildTypeOptions('release'),
+ JavaSpecificOptions(),
+ ],
+ visible=False,
+ )
+ self['licenses'] = OptsHandler(
+ action=app.execute(action=do_licenses),
+ description='Print known licenses grouped by their properties',
+ opts=self.common_opts
+ + self.common_build_facade_opts()
+ + [
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ DumpLicenseOptions(),
+ ],
+ visible=False,
+ )
+ self['forced-deps'] = OptsHandler(
+ action=app.execute(action=do_forced_deps),
+ description='Print known forced dependency management',
+ opts=FullForcedDepsOptions(),
+ visible=False,
+ )
+ self['conf-docs'] = OptsHandler(
+ action=app.execute(action=do_conf_docs),
+ description='Print descriptions of entities (modules, macros, multimodules, etc.)',
+ opts=self.common_opts + self.common_build_facade_opts() + [DumpDescriptionOptions()],
+ )
+ self['root'] = OptsHandler(
+ action=app.execute(lambda params: sys.stdout.write(params.arc_root) and 0),
+ description='Print Arcadia root',
+ opts=self.common_opts,
+ )
+ self['vcs-info'] = OptsHandler(
+ action=app.execute(action=do_dump_vcs_info),
+ description='Print VCS revision information.',
+ opts=self.common_opts + self.common_build_facade_opts(),
+ visible=False,
+ )
+ self['raw-vcs-info'] = OptsHandler(
+ action=app.execute(action=do_dump_raw_vcs_info),
+ description='Print VCS revision information.',
+ opts=self.common_opts + self.common_build_facade_opts(),
+ visible=False,
+ )
+ self['svn-revision'] = OptsHandler(
+ action=app.execute(action=do_dump_svn_revision),
+ description='Print SVN revision information.',
+ opts=self.common_opts + self.common_build_facade_opts(),
+ visible=False,
+ )
+ self['recipes'] = OptsHandler(
+ action=app.execute(action=do_recipes),
+ description='All recipes used in tests',
+ opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions(), DumpRecipesOptions()],
+ )
+ if app_config.in_house:
+ import devtools.ya.handlers.dump.arcadia_specific as arcadia_specific
+ from handlers.dump.debug import debug_handler
+
+ self['groups'] = arcadia_specific.GroupsHandler()
+ self['atd-revisions'] = OptsHandler(
+ action=app.execute(action=arcadia_specific.do_atd_revisions),
+ description='Dump revisions of trunk/arcadia_tests_data',
+ opts=self.common_opts
+ + [DumpAtdRevisionOptions(), CustomFetcherOptions(), SandboxAuthOptions(), ToolsOptions()],
+ )
+ self['debug'] = debug_handler
+
+
+def _do_dump(gen_func, params, debug_options=[], write_stdout=True, build_root=None, **kwargs):
+ for name in 'debug_options', 'filter_opts', 'peerdir_opts':
+ if hasattr(params, name):
+ debug_options.extend(getattr(params, name))
+ logger.debug('abs_targets: %s', params.abs_targets)
+ with temp_dir() as tmp:
+ res = gen_func(
+ build_root=build_root or tmp,
+ build_type=params.build_type,
+ build_targets=params.abs_targets,
+ debug_options=debug_options,
+ flags=params.flags,
+ ymake_bin=getattr(params, 'ymake_bin', None),
+ host_platform=params.host_platform,
+ target_platforms=params.target_platforms,
+ **kwargs
+ )
+ if write_stdout:
+ sys.stdout.write(res.stdout)
+ return res
+
+
+def do_modules(params):
+ _do_dump(gen_modules, params)
+
+
+def do_module_info(params, write_stdout=True):
+ if params.with_data:
+ params.flags['YMAKE_ADD_DATA'] = 'yes'
+ return _do_dump(
+ gen_func=gen_module_info,
+ params=params,
+ modules_info_filter=getattr(params, 'modules_info_filter'),
+ modules_info_file=getattr(params, 'modules_info_file'),
+ write_stdout=write_stdout,
+ )
+
+
+def do_dir_graph(params):
+ def merge_lists(one, two):
+ return sorted(set(one + two))
+
+ def get_test_dependencies(deps, arc_root, test, test_data=None):
+ data = merge_lists(test_data or [], deps)
+ return merge_lists(data, get_canondata_paths(arc_root, test))
+
+ if params.dump_deps:
+ deps = _do_dump(gen_dependencies, params, params.legacy_deps_opts, write_stdout=False)
+ json.dump(json.loads(deps.stdout), sys.stdout, indent=4, sort_keys=True)
+ return
+
+ if params.dump_reachable_dirs:
+ _do_dump(gen_graph, params, params.legacy_deps_opts + ['x', 'M'])
+ return
+
+ dg = _do_dump(
+ gen_dir_graph, params, params.legacy_deps_opts, write_stdout=False, split_by_types=params.split_by_type
+ )
+
+ tests = get_tests(params)
+ for test in tests:
+ test_data = test.get_arcadia_test_data()
+ if test.project_path in test_data:
+ test_data.remove(test.project_path)
+ if params.split_by_type:
+ if test.project_path not in dg:
+ dg[test.project_path] = {}
+ dg[test.project_path]["INCLUDE"] = get_test_dependencies(
+ dg[test.project_path].get("INCLUDE", []), params.arc_root, test
+ )
+ dg[test.project_path]["DATA"] = merge_lists(dg[test.project_path].get("DATA", []), test_data)
+ else:
+ dg[test.project_path] = get_test_dependencies(
+ dg.get(test.project_path, []), params.arc_root, test, test_data
+ )
+
+ if params.trace_from is not None:
+ print('\n'.join(reachable(dg, params.trace_from, params.split_by_type)))
+
+ elif params.plain_dump:
+
+ def get_plain_deps():
+ plain_deps = set()
+ for k, v in six.iteritems(dg):
+ plain_deps.add(k)
+ plain_deps |= set(v)
+ return sorted(list(plain_deps))
+
+ json.dump(get_plain_deps(), sys.stdout, indent=4, sort_keys=True)
+ else:
+ json.dump(dg, sys.stdout, indent=4, sort_keys=True)
+
+
+def get_canondata_paths(arc_root, test):
+ paths = []
+ canondata_path = os.path.join(arc_root, test.project_path, "canondata")
+ if os.path.exists(canondata_path):
+ for root, dirs, files in os.walk(canondata_path):
+ paths.append(os.path.relpath(root, arc_root))
+ return paths
+
+
+DUMP_OPTS_GROUP = Group("Dump options", 0)
+
+
+class SplitByTypeOptions(Options):
+ def __init__(self):
+ self.split_by_type = False
+
+ @staticmethod
+ def consumer():
+ return ArgConsumer(
+ ['--split'],
+ help='split by type',
+ hook=SetConstValueHook('split_by_type', True),
+ group=DUMP_OPTS_GROUP,
+ )
+
+
+class DumpDirOptions(Options):
+ def __init__(self):
+ self.trace_from = None
+ self.plain_dump = False
+ self.dump_deps = False
+ self.dump_reachable_dirs = False
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--trace'],
+ help='trace from sources',
+ hook=SetValueHook('trace_from'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--plain'],
+ help='plain dump mode',
+ hook=SetConstValueHook('plain_dump', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--dump-deps'],
+ help='Dump dependencies (legacy)',
+ hook=SetConstValueHook('dump_deps', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--reachable-dirs'],
+ help='Dump reachable dirs',
+ hook=SetConstValueHook('dump_reachable_dirs', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class PeerDirectoriesOptions(Options):
+ def __init__(self):
+ self.peerdir_opts = []
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--old-peerdirs'],
+ help='Include indirect (module->dir->module) PEERDIRs',
+ hook=SetConstAppendHook('peerdir_opts', 'I'),
+ group=DUMP_OPTS_GROUP,
+ visible=False,
+ )
+ ]
+
+
+class LegacyDepsOptions(Options):
+ def __init__(self):
+ self.legacy_deps_opts = ['I']
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--no-legacy-deps'],
+ help='Exclude legacy dependencies (inderect PEERDIR relations)',
+ hook=SetConstValueHook('legacy_deps_opts', []),
+ group=DUMP_OPTS_GROUP,
+ )
+ ]
+
+
+class DumpSrcDepsOptions(Options):
+ def __init__(self):
+ self.with_yamake = False
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--with-yamakes'],
+ help='Include ya.make files, that are only used as a build configuration',
+ hook=SetConstValueHook('with_yamake', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DepFilterOptions(Options):
+ def __init__(self):
+ self.filter_opts = []
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--no-tools'],
+ help='Exclude tools dependencies',
+ hook=SetConstAppendHook('filter_opts', 'V'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--no-addincls'],
+ help='Exclude ADDINCLs dependencies',
+ hook=SetConstAppendHook('filter_opts', 'T'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DepGraphOutputType(enum.Enum):
+ CUSTOM = 0
+ FLAT_JSON = 1
+ FLAT_JSON_FILES = 2
+
+
+class DepGraphOutputOptions(Options):
+ def __init__(self):
+ self.output_type = DepGraphOutputType.CUSTOM
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--flat-json'],
+ help='Dump dep-graph in flat json format',
+ hook=SetConstValueHook('output_type', DepGraphOutputType.FLAT_JSON),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--flat-json-files'],
+ help='Dump dep-graph in flat json format without commands',
+ hook=SetConstValueHook('output_type', DepGraphOutputType.FLAT_JSON_FILES),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DepTraverseOptions(Options):
+ @staticmethod
+ def consumer():
+ def test_depends_updater(flags):
+ flags['TRAVERSE_DEPENDS'] = 'yes'
+ flags['TRAVERSE_RECURSE_FOR_TESTS'] = 'yes'
+ return flags
+
+ def ignore_recurses_updater(flags):
+ flags['TRAVERSE_RECURSE'] = 'no'
+ return flags
+
+ # Hooks update flags property from FlagsOptions
+ return [
+ ArgConsumer(
+ ['-t', '--force-build-depends'],
+ help='Include DEPENDS and RECURSE_FOR_TESTS dependencies',
+ hook=UpdateValueHook('flags', test_depends_updater),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['-A'],
+ help='Same as -t',
+ hook=UpdateValueHook('flags', test_depends_updater),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--ignore-recurses'],
+ help='Exclude all RECURSE dependencies',
+ hook=UpdateValueHook('flags', ignore_recurses_updater),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DataOptions(Options):
+ def __init__(self):
+ self.with_data = False
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--with-data'],
+ help='Include DATA files and dirs',
+ hook=SetConstValueHook('with_data', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DumpTestListOptions(Options):
+ def __init__(self):
+ self.follow_dependencies = False
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--skip-deps'],
+ help='Now default and ignored',
+ hook=SetConstValueHook('follow_dependencies', False),
+ group=DUMP_OPTS_GROUP,
+ visible=False,
+ deprecated=True,
+ ),
+ ]
+
+
+class DumpLicenseOptions(Options):
+ def __init__(self):
+ self.json_licenses = False
+ self.link_type = 'static'
+ self.custom_tags = []
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--json'],
+ help='Machine oriented json representation',
+ hook=SetConstValueHook('json_licenses', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--link-type'],
+ help='Specify library link type (static|dynamic)',
+ hook=SetValueHook('link_type'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--custom-license-tag'],
+ help='Specify library link type (static|dynamic)',
+ hook=SetAppendHook('custom_tags'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DumpForcedDepsOptions(Options):
+ def __init__(self):
+ self.json_forced_deps = False
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--json'],
+ help='Json forced dependency representation',
+ hook=SetConstValueHook('json_forced_deps', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+def FullForcedDepsOptions():
+ return (
+ DumpYaHandler.common_opts
+ + DumpYaHandler.common_build_facade_opts()
+ + [
+ YMakeDebugOptions(),
+ CustomBuildRootOptions(),
+ DumpForcedDepsOptions(),
+ ]
+ )
+
+
+class RelationSrcDstOptions(Options):
+ def __init__(self, free_dst=False):
+ self._free_dst = free_dst
+ self.recursive = False
+ self.relation_src = []
+ self.relation_dst = []
+
+ def consumer(self):
+ res = [
+ ArgConsumer(
+ ['--from'],
+ help='Dump relations from this target (path relative to the arcadia root)',
+ hook=SetAppendHook('relation_src'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--to'],
+ help='Dump relations to this target (path relative to the arcadia root)',
+ hook=SetAppendHook('relation_dst'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--recursive'],
+ help='Show relations between RELATION_SRC and all modules from RELATION_DST directories',
+ hook=SetConstValueHook('recursive', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+ if self._free_dst:
+ res.append(
+ FreeArgConsumer(
+ help='relation destination target',
+ hook=ExtendHook('relation_dst'),
+ )
+ )
+ return res
+
+ def postprocess(self):
+ if len(self.relation_dst) == 0:
+ raise ArgsValidatingException('Error: no target is set')
+
+
+class FilesFilterOptions(Options):
+ def __init__(self):
+ self.skip_make_files = False
+ self.mark_make_files = False
+
+ def consumer(self):
+ res = [
+ ArgConsumer(
+ ['--skip-make-files'],
+ help='Skip all make files',
+ hook=SetConstValueHook('skip_make_files', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--mark-make-files'],
+ help='Mark all make files as Makefile',
+ hook=SetConstValueHook('mark_make_files', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+ return res
+
+
+class DumpAllRelationsOptions(Options):
+ def __init__(self):
+ self.json_format = False
+ self.show_targets_deps = False
+
+ def consumer(self):
+ res = [
+ ArgConsumer(
+ ['--json'],
+ help='Dump relations in json format',
+ hook=SetConstValueHook('json_format', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--show-targets-deps'],
+ help='Show dependencies between RELATION_DST targets',
+ hook=SetConstValueHook('show_targets_deps', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+ return res
+
+
+class DumpDescriptionOptions(Options):
+ def __init__(self):
+ self.dump_all_conf_docs = False
+ self.conf_docs_json = False
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--dump-all'],
+ help='Dump information for all entities including internal',
+ hook=SetConstValueHook('dump_all_conf_docs', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--json'],
+ help='Dump information for all entities including internal in json format, uses are included',
+ hook=SetConstValueHook('conf_docs_json', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DumpAtdRevisionOptions(Options):
+ def __init__(self):
+ self.human_readable = False
+ self.print_size = False
+ self.arcadia_revision = None
+ self.path = None
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--human-readable'],
+ help='Add human-readable paths to comments',
+ hook=SetConstValueHook('human_readable', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--print-size'],
+ help='Add human-readable paths to comments',
+ hook=SetConstValueHook('print_size', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--revision'],
+ help='Gather revisions for this arcadia revision',
+ hook=SetValueHook('arcadia_revision'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--path'],
+ help='Gather revisions for this path recursively',
+ hook=SetValueHook('path'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DumpModuleInfoOptions(Options):
+ def __init__(self):
+ self.modules_info_file = ''
+ self.modules_info_filter = ''
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--modules-info-file'],
+ help='Save modules informaton into file specified',
+ hook=SetValueHook('modules_info_file'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ArgConsumer(
+ ['--modules-info-filter'],
+ help='Dump information only for modules matching regexp',
+ hook=SetValueHook('modules_info_filter'),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+
+
+class DumpRecipesOptions(Options):
+ def __init__(self):
+ self.json_format = False
+
+ def consumer(self):
+ res = [
+ ArgConsumer(
+ ['--json'],
+ help='Dump recipes in json format',
+ hook=SetConstValueHook('json_format', True),
+ group=DUMP_OPTS_GROUP,
+ ),
+ ]
+ return res
+
+
+def do_dump_srcdeps(params):
+ debug_options = []
+ params.flags['YMAKE_ADD_DATA'] = 'yes'
+ if params.with_yamake:
+ debug_options.append('mkf')
+ _do_dump(gen_srcdeps, params, debug_options)
+
+
+def do_dot_graph(params):
+ _do_dump(gen_modules, params, ['D'])
+
+
+def do_dep_graph(params):
+ options = params.legacy_deps_opts
+ if params.output_type == DepGraphOutputType.FLAT_JSON:
+ options.append('flat-json-with-cmds')
+ elif params.output_type == DepGraphOutputType.FLAT_JSON_FILES:
+ options.append('flat-json')
+ _do_dump(gen_graph, params, options)
+
+
+def do_json_dep_graph(params):
+ _do_dump(gen_json_graph, params, params.legacy_deps_opts)
+
+
+def do_licenses(params):
+ _do_dump(
+ gen_licenses_list,
+ params,
+ write_stdout=True,
+ debug_options=['lic-json' if params.json_licenses else 'lic'],
+ lic_link_type=params.link_type,
+ lic_custom_tags=params.custom_tags,
+ )
+
+
+def do_forced_deps(params, write_stdout=True):
+ res = _do_dump(
+ gen_forced_deps_list,
+ params,
+ write_stdout=write_stdout,
+ debug_options=['fdm-json' if params.json_forced_deps else 'fdm'],
+ )
+ return '' if write_stdout else res.stdout
+
+
+# TODO: support host/target platforms opts
+def do_build_plan(params):
+ with temp_dir() as tmp:
+ graph = gen_plan(
+ arc_root=params.arc_root,
+ build_root=tmp,
+ build_type=params.build_type,
+ build_targets=params.abs_targets,
+ debug_options=params.debug_options,
+ flags=params.flags,
+ ymake_bin=params.ymake_bin,
+ no_ymake_resource=params.no_ymake_resource,
+ vcs_file=params.vcs_file,
+ )
+ json.dump(graph, sys.stdout, indent=4, sort_keys=True)
+
+
+def do_relation(params):
+ options = []
+ if params.recursive:
+ options.append('recursive')
+ for dst in params.relation_dst:
+ _do_dump(
+ gen_relation,
+ params,
+ options,
+ find_path_from=params.relation_src,
+ find_path_to=[dst],
+ )
+
+
+def do_all_relations(params):
+ options = ['J' if params.json_format else 'D']
+ if params.recursive:
+ options.append('recursive')
+ if params.show_targets_deps:
+ options.append('show-targets-deps')
+ _do_dump(
+ gen_relation,
+ params,
+ options,
+ find_path_from=params.relation_src,
+ find_path_to=params.relation_dst,
+ )
+
+
+def do_files(params):
+ options = []
+ if params.with_data:
+ options = ['dump-data']
+ params.flags['YMAKE_ADD_DATA'] = 'yes'
+ if params.skip_make_files:
+ options.append('skip-make-files')
+ if params.mark_make_files:
+ options.append('mark-make-files')
+ _do_dump(gen_filelist, params, options)
+
+
+def do_loops(params):
+ _do_dump(gen_all_loops, params)
+
+
+def do_peerdir_loops(params):
+ _do_dump(gen_dir_loops, params)
+
+
+def do_imprint(params):
+ lines = imprint.generate_detailed_imprints(os.curdir)
+ writer = csv.writer(sys.stdout, delimiter='\t')
+ for line in lines:
+ writer.writerow(line)
+
+
+# TODO: support host/target platforms opts
+def do_uids(params):
+ uids = gen_uids(
+ arc_root=params.arc_root,
+ build_root=None,
+ build_type=params.build_type,
+ build_targets=params.abs_targets,
+ debug_options=[],
+ flags={},
+ ymake_bin=None,
+ )
+ json.dump(uids, sys.stdout, indent=4, sort_keys=True)
+
+
+def get_tests(params):
+ params.flags['TRAVERSE_RECURSE_FOR_TESTS'] = 'yes'
+ kwargs = {
+ 'build_root': params.bld_root,
+ 'build_type': params.build_type,
+ 'abs_targets': params.abs_targets,
+ 'debug_options': [],
+ 'flags': params.flags,
+ 'ymake_bin': params.ymake_bin,
+ 'arcadia_tests_data_path': 'arcadia_tests_data',
+ }
+ (
+ _,
+ test_dart,
+ ) = gen_test_dart(**kwargs)
+
+ return sorted(
+ generate_tests_by_dart(
+ test_dart,
+ opts=params,
+ ),
+ key=lambda test: os.path.join(test.project_path, test.name),
+ )
+
+
+def do_json_test_list(params):
+ json.dump([t.save() for t in get_tests(params)], sys.stdout, indent=4, sort_keys=True)
+
+
+def do_test_list(params):
+ print(pickle.dumps(get_tests(params)))
+
+
+def do_conf(params):
+ with temp_dir() as tmp:
+ generation_conf = _do_dump(gen_conf, params, write_stdout=False, build_root=tmp)
+ print(open(generation_conf, 'r').read())
+
+
+def do_conf_docs(params):
+ _do_dump(dump_mmm_docs, params, dump_all_conf_docs=params.dump_all_conf_docs, conf_docs_json=params.conf_docs_json)
+
+
+def do_dump_raw_vcs_info(params):
+ sys.stdout.write(json.dumps(vcsversion.get_raw_version_info(params.arc_root, params.bld_root)))
+
+
+def do_dump_vcs_info(params):
+ fake_data = strtobool(params.flags.get('NO_VCS_DEPENDS', 'no'))
+ fake_build_info = strtobool(params.flags.get('CONSISTENT_BUILD', 'no'))
+ sys.stdout.write(vcsversion.get_version_info(params.arc_root, params.bld_root, fake_data, fake_build_info) + '\n')
+
+
+def do_dump_svn_revision(params):
+ sys.stdout.write(str(vcsversion.repo_config(params.arc_root)))
+
+
+def do_recipes(params):
+ tests = get_tests(params)
+ test_recipes = []
+ for test in tests:
+ encoded_recipes = test.recipes
+ if not encoded_recipes:
+ continue
+ decoded_recipes = decode_recipe_cmdline(encoded_recipes)
+ test_recipes.append(decoded_recipes)
+
+ recipes = set()
+ for test_recipe in test_recipes:
+ for recipe in test_recipe:
+ recipes.add(tuple(recipe))
+
+ if params.json_format:
+ output = json.dumps(recipes, indent=4, sort_keys=True)
+ else:
+ recipes_str = [' '.join(recipe) for recipe in recipes]
+ output = ' '.join(recipes_str)
+ print(output)
diff --git a/devtools/ya/handlers/dump/gen_conf_docs.py b/devtools/ya/handlers/dump/gen_conf_docs.py
new file mode 100644
index 0000000000..4b9978f226
--- /dev/null
+++ b/devtools/ya/handlers/dump/gen_conf_docs.py
@@ -0,0 +1,384 @@
+from __future__ import absolute_import, unicode_literals
+import exts.yjson as json
+import re
+import os
+import six
+
+import core.config
+
+import build.genconf
+import build.ymake2
+
+from yalibrary.vcs import vcsversion
+from build.build_facade import gen_conf
+from exts.strtobool import strtobool
+
+
+class _Markdown:
+ header = '#'
+ alink = 'https://a.yandex-team.ru/arc/trunk/arcadia/'
+ # Section nesting count
+ scount = 2
+ # Description nesting count
+ dcount = 6
+ # Avoid too many entries in toc for macros
+ chunks_in_toc = 10
+ # Checks description
+ internal_pattern = re.compile(r'^\s*@?internal[.]?\s*$', flags=re.IGNORECASE)
+ # Checks description
+ deprecated_pattern = re.compile(r'^\s*@?deprecated[.]?\s*$', flags=re.IGNORECASE)
+ # Checks header
+ h_patterns = {
+ 'internal': re.compile('.*#.*internal.*', flags=re.IGNORECASE),
+ 'deprecated': re.compile('.*#.*deprecated.*', flags=re.IGNORECASE),
+ }
+
+ def __init__(self, arc_root, dump_all_descs, use_svn):
+ self.descs = {'macros': {}, 'modules': {}, 'multimodules': {}, 'unknowns': {}}
+ self.links = {}
+ self.anchors = {}
+ try:
+ self.svn_revision, _ = vcsversion.repo_config(arc_root) if use_svn else (-1, '')
+ except Exception:
+ self.svn_revision = -1
+ self.dump_all_descs = dump_all_descs
+
+ def process_entry(self, file, entry):
+ if 'kind' not in entry or entry['kind'] != 'node':
+ return
+
+ self._add_entry_optionally(file, entry)
+
+ def dump(self):
+ res = self._dump_toc()
+
+ for type in ['multimodules', 'modules', 'macros', 'unknowns']:
+ if self.descs[type]:
+ res += _Markdown._format_section(type)
+
+ for name in sorted(self.descs[type]):
+ for d in self.descs[type][name]['text']:
+ res += six.ensure_str(d)
+
+ for type in ['multimodules', 'modules', 'macros', 'unknowns']:
+ if self.descs[type]:
+ for name in sorted(self.descs[type]):
+ link = self.descs[type][name]['link']
+ res += six.ensure_str(link)
+
+ return res
+
+ def _add_entry_optionally(self, file, entry):
+ props = entry['properties']
+ doc = {'name': props['name'], 'type': props['type'], 'file': file, 'line': entry['range']['line']}
+
+ if 'comment' in props:
+ doc['long desc'] = props['comment']
+ if 'usage' in props:
+ doc['usage'] = props['usage']
+
+ doc['revision'] = self.svn_revision
+
+ descs, link = self._format_entry(doc)
+
+ dictionary = (
+ self.descs[doc['type'] + 's']
+ if doc['type'] in ['macro', 'module', 'multimodule']
+ else self.descs['unknowns']
+ )
+
+ if not self.dump_all_descs and _Markdown._is_internal(descs[0]):
+ return
+
+ dictionary[doc['name']] = {'text': descs, 'link': link, 'src_data': doc}
+
+ @classmethod
+ def _is_internal(cls, header):
+ return cls.h_patterns['internal'].match(header)
+
+ def _dump_toc(self):
+ res = '*Do not edit, this file is generated from comments to macros definitions using `ya dump conf-docs{all}`.*\n\n'.format(
+ all=' --dump-all' if self.dump_all_descs else ''
+ )
+ res += '{markup} ya.make {all}commands\n\n'.format(
+ markup=_Markdown.header, all='and core.conf ' if self.dump_all_descs else ''
+ )
+ res += (
+ 'General info: [How to write ya.make files](https://wiki.yandex-team.ru/yatool/HowToWriteYaMakeFiles)\n\n'
+ )
+ res += '{markup} Table of contents\n\n'.format(markup=_Markdown.header * 2)
+
+ for type in ['multimodules', 'modules', 'macros', 'unknowns']:
+ if self.descs[type]:
+ res += _Markdown._format_toc_section(type)
+ if type != 'macros':
+ for name in sorted(self.descs[type]):
+ res += _Markdown._format_toc_header(self.descs[type][name]['src_data'])
+ else:
+ chunk_cnt = 0
+ first_macro = {}
+ last_macro = {}
+ for name in sorted(self.descs[type]):
+ if chunk_cnt == 0:
+ first_macro = self.descs[type][name]['src_data']
+ chunk_cnt += 1
+ last_macro = self.descs[type][name]['src_data']
+
+ if chunk_cnt == _Markdown.chunks_in_toc:
+ chunk_cnt = 0
+ res += _Markdown._format_toc_macro_header(first_macro, last_macro)
+
+ if chunk_cnt != 0:
+ res += _Markdown._format_toc_macro_header(first_macro, last_macro)
+ return res
+
+ @classmethod
+ def _format_entry(cls, doc):
+ descs = []
+
+ lines, special_tags = _Markdown._format_desc(doc)
+ descs.append(_Markdown._format_header(doc, special_tags))
+ descs.append(lines)
+ return (descs, _Markdown._format_link(doc))
+
+ @classmethod
+ def _format_toc_section(cls, type):
+ return '{indent} * [{section}](#{anchor})\n'.format(
+ indent=' ' * cls.scount, section=type.capitalize(), anchor=type
+ )
+
+ @classmethod
+ def _format_section(cls, type):
+ return '{markup} {text} <a name="{anchor}"></a>\n\n'.format(
+ markup=cls.header * cls.scount, text=type.capitalize(), anchor=type
+ )
+
+ @staticmethod
+ def _format_header_anchor(doc):
+ return '<a name="{atype}_{aname}"></a>'.format(atype=doc['type'], aname=doc['name'])
+
+ @classmethod
+ def _format_toc_header(cls, doc):
+ qual = doc['type'].capitalize() if doc['type'] in ['macro', 'module', 'multimodule'] else "Unknown"
+ return '{indent} - {qual} [{name}](#{type}_{name})\n'.format(
+ indent=' ' * cls.dcount, qual=qual, name=doc['name'], type=doc['type']
+ )
+
+ @classmethod
+ def _format_toc_macro_header(cls, fdoc, ldoc):
+ return '{indent} - Macros [{fname}](#{ftype}_{fname}) .. [{lname}](#{ltype}_{lname})\n'.format(
+ indent=' ' * cls.dcount, fname=fdoc['name'], ftype=fdoc['type'], lname=ldoc['name'], ltype=ldoc['type']
+ )
+
+ # Also adds special tags 'internal' and 'deprecated'
+ @classmethod
+ def _format_header(cls, doc, special_tags):
+ name = doc['name']
+ usage = doc['usage'] if 'usage' in doc else ""
+ usage = name if not usage else usage
+
+ qual = doc['type'].capitalize() if doc['type'] in ['macro', 'module', 'multimodule'] else "Unknown"
+
+ usage = _Markdown._remove_formatting(usage.rstrip().lstrip())
+ name = _Markdown._remove_formatting(name)
+
+ usage = usage.replace(name, '[' + name + "][]", 1)
+
+ if special_tags:
+ special = ''
+ for tag in special_tags:
+ if not cls.h_patterns[tag].match(usage):
+ special += ' ' + tag
+ # Emphasis
+ if usage.find("#") != -1:
+ usage += special
+ else:
+ usage += ' #' + special[1:]
+
+ # Emphasis
+ if usage.find("#") != -1:
+ usage = usage.replace('#', "_#", 1)
+ usage += "_"
+
+ return '{markup} {type} {usage} {anchor}\n'.format(
+ markup=cls.header * cls.dcount, type=qual, usage=usage, anchor=_Markdown._format_header_anchor(doc)
+ )
+
+ # Prints verbatim. Strips unnecessary indent and escapes '_'/'*'.
+ @classmethod
+ def _format_desc(cls, doc):
+ result = ""
+
+ desc = ""
+ if 'long desc' in doc:
+ desc = doc['long desc'].rstrip()
+ if not desc:
+ desc = " Not documented yet.\n"
+
+ lines = _Markdown._strip_blanks(desc.splitlines())
+ lines = _Markdown._remove_formatting_markdown(lines)
+
+ result += '\n'.join(lines) + "\n\n"
+
+ special_tags = []
+ if doc['name'].startswith('_') or any([cls.internal_pattern.match(x) for x in lines]):
+ special_tags.append("internal")
+ if any([cls.deprecated_pattern.match(x) for x in lines]):
+ special_tags.append("deprecated")
+
+ return (result, special_tags)
+
+ @staticmethod
+ def _format_link(doc):
+ return ' [{tag_name}]: {baselink}{file}{rev}#L{line}\n'.format(
+ tag_name=_Markdown._remove_formatting(doc['name']),
+ baselink=_Markdown.alink,
+ file=doc['file'],
+ rev='?rev=' + str(doc['revision']) if doc['revision'] > 0 else '',
+ line=doc['line'],
+ )
+
+ @staticmethod
+ def _strip_blanks(lines):
+ first = 0
+ for line in lines:
+ if not line.lstrip().rstrip():
+ first += 1
+ else:
+ break
+ last = 0
+ for line in reversed(lines):
+ if not line.lstrip().rstrip():
+ last += 1
+ else:
+ break
+ lines = lines[first : (len(lines) - last)]
+ lines = [x.rstrip().expandtabs(4) for x in lines]
+ indent = 10000
+ for line in lines:
+ if line:
+ indent = min(indent, len(line) - len(line.lstrip()))
+
+ if indent > 0:
+ lines = [x.replace(' ' * indent, '', 1) for x in lines]
+
+ return lines
+
+ # Conditionally removed formatting.
+ # Code blocks are not modified
+ @staticmethod
+ def _remove_formatting_markdown(lines):
+ code = False
+ new_paragraph = True
+ backtick_block = False
+
+ res = []
+ for line in lines:
+ if new_paragraph:
+ if line.startswith('\t') or line.startswith(' ' * 4):
+ code = True
+
+ if not line.startswith('\t') and not line.startswith(' ' * 4):
+ code = False
+
+ new_paragraph = not line
+
+ if line.lstrip() == '```':
+ backtick_block = not backtick_block
+
+ res.append(line if code or backtick_block else _Markdown._remove_formatting(line))
+ return res
+
+ # Unconditionally removes formatting due to '_'/'*'
+ @staticmethod
+ def _remove_formatting(x):
+ return x.replace("_", r"\_").replace("*", r"\*")
+
+
+def _gen(
+ custom_build_directory,
+ build_type,
+ build_targets,
+ debug_options,
+ flags=None,
+ warn_mode=None,
+ ymake_bin=None,
+ platform=None,
+ host_platform=None,
+ target_platforms=None,
+ **kwargs
+):
+ generation_conf = gen_conf(
+ build_root=custom_build_directory,
+ build_type=build_type,
+ build_targets=build_targets,
+ flags=flags,
+ host_platform=host_platform,
+ target_platforms=target_platforms,
+ )
+ res, evlog_dump = build.ymake2.ymake_dump(
+ custom_build_directory=custom_build_directory,
+ build_type=build_type,
+ abs_targets=build_targets,
+ debug_options=debug_options,
+ warn_mode=warn_mode,
+ flags=flags,
+ ymake_bin=ymake_bin,
+ platform=platform,
+ grab_stderr=True,
+ custom_conf=generation_conf,
+ **kwargs
+ )
+ return res
+
+
+def dump_mmm_docs(
+ build_root,
+ build_type,
+ build_targets,
+ debug_options,
+ flags,
+ dump_all_conf_docs=None,
+ conf_docs_json=None,
+ ymake_bin=None,
+ platform=None,
+ host_platform=None,
+ target_platforms=None,
+):
+ json_dump_name = os.path.join(build_root, 'ymake.dump.ydx.json')
+ arc_root = core.config.find_root_from(build_targets)
+ null_ya_make = os.path.join(arc_root, 'build', 'docs', 'empty')
+
+ if not conf_docs_json:
+ if not os.path.exists(null_ya_make):
+ raise "Empty project not found, dump conf-docs may work too long"
+
+ res = _gen(
+ custom_build_directory=build_root,
+ build_type=build_type,
+ # Override target
+ build_targets=[null_ya_make] if not conf_docs_json else build_targets,
+ debug_options=debug_options,
+ flags=flags,
+ ymake_bin=ymake_bin,
+ platform=platform,
+ host_platform=host_platform,
+ target_platforms=target_platforms,
+ yndex_file=json_dump_name,
+ )
+
+ if conf_docs_json:
+ with open(json_dump_name, 'r') as jfile:
+ res.stdout += jfile.read()
+ else:
+ with open(json_dump_name, 'r') as jfile:
+ contents = jfile.read()
+ jdata = json.loads(contents)
+ no_svn = True if 'NO_SVN_DEPENDS' in flags and strtobool(flags['NO_SVN_DEPENDS']) else False
+ doc = _Markdown(arc_root, dump_all_conf_docs, not no_svn)
+ for efile in jdata:
+ for entry in jdata[efile]:
+ doc.process_entry(efile, entry)
+ res.stdout += doc.dump()
+
+ return res
diff --git a/devtools/ya/handlers/dump/ya.make b/devtools/ya/handlers/dump/ya.make
new file mode 100644
index 0000000000..adbc66ad9f
--- /dev/null
+++ b/devtools/ya/handlers/dump/ya.make
@@ -0,0 +1,52 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.dump
+ __init__.py
+ gen_conf_docs.py
+)
+
+PEERDIR(
+ contrib/python/pathlib2
+ contrib/python/six
+ contrib/python/toml
+ devtools/ya/app
+ devtools/ya/build
+ devtools/ya/build/build_facade
+ devtools/ya/build/build_opts
+ devtools/ya/build/genconf
+ devtools/ya/build/ymake2
+ devtools/ya/core
+ devtools/ya/core/config
+ devtools/ya/core/imprint
+ devtools/ya/core/yarg
+ devtools/ya/exts
+ devtools/ya/test/dartfile
+ devtools/ya/test/explore
+ devtools/ya/yalibrary/debug_store
+ devtools/ya/yalibrary/debug_store/processor
+ devtools/ya/yalibrary/debug_store/store
+ devtools/ya/yalibrary/tools
+ devtools/ya/yalibrary/vcs
+ devtools/ya/yalibrary/vcs/vcsversion
+ devtools/ya/yalibrary/yandex/sandbox/misc
+ library/python/fs
+ library/python/func
+ library/python/tmp
+)
+
+IF (NOT YA_OPENSOURCE)
+ PEERDIR(
+ devtools/ya/handlers/dump/arcadia_specific
+ devtools/ya/handlers/dump/debug
+ )
+ENDIF()
+
+END()
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/gc/__init__.py b/devtools/ya/handlers/gc/__init__.py
new file mode 100644
index 0000000000..824a35edf2
--- /dev/null
+++ b/devtools/ya/handlers/gc/__init__.py
@@ -0,0 +1,342 @@
+from __future__ import absolute_import
+import glob
+import os
+import logging
+import sys
+import time
+import six
+
+import app
+import core.yarg
+import core.common_opts
+import core.config as cc
+import build.ya_make as ym
+from exts import fs
+from core.common_opts import CustomBuildRootOptions
+from build.build_opts import LocalCacheOptions, DistCacheSetupOptions, parse_size_arg, parse_timespan_arg
+from exts.windows import on_win
+from yalibrary.runner import result_store
+import yalibrary.toolscache as tc
+
+if six.PY3:
+ long = int
+
+logger = logging.getLogger(__name__)
+
+
+def _to_size_in_gb(size):
+ try:
+ return int(float(size) * 1024 * 1024 * 1024)
+ except ValueError:
+ pass
+
+ return parse_size_arg(size)
+
+
+def _to_size_in_mb(size):
+ try:
+ return int(float(size) * 1024 * 1024)
+ except ValueError:
+ pass
+
+ return parse_size_arg(size)
+
+
+def _to_timespan_in_hours(timespan):
+ try:
+ return int(float(timespan) * 60 * 60)
+ except ValueError:
+ pass
+
+ return parse_timespan_arg(timespan)
+
+
+class CollectCacheOptions(LocalCacheOptions):
+ def __init__(self):
+ super(CollectCacheOptions, self).__init__()
+ self.object_size_limit = None
+ self.age_limit = None
+ self.symlinks_ttl = 0
+
+ def consumer(self):
+ return super(CollectCacheOptions, self).consumer() + [
+ core.yarg.ArgConsumer(
+ ['--size-limit'],
+ help='Strip build cache to size (in GiB if not set explicitly)',
+ hook=core.yarg.SetValueHook(
+ 'cache_size',
+ transform=_to_size_in_gb,
+ default_value=lambda x: str(_to_size_in_gb(x) * 1.0 / 1024 / 1024 / 1024),
+ ),
+ group=core.yarg.BULLET_PROOF_OPT_GROUP,
+ visible=False,
+ ),
+ core.yarg.ArgConsumer(
+ ['--object-size-limit'],
+ help='Strip build cache from large objects (in MiB if not set explicitly)',
+ hook=core.yarg.SetValueHook('object_size_limit', transform=_to_size_in_mb),
+ group=core.yarg.BULLET_PROOF_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--age-limit'],
+ help='Strip build cache from old objects (in hours if not set explicitly)',
+ hook=core.yarg.SetValueHook('age_limit', transform=_to_timespan_in_hours),
+ group=core.yarg.BULLET_PROOF_OPT_GROUP,
+ ),
+ ]
+
+
+class GarbageCollectionYaHandler(core.yarg.CompositeHandler):
+ def __init__(self):
+ core.yarg.CompositeHandler.__init__(self, description='Collect garbage')
+ self['cache'] = core.yarg.OptsHandler(
+ action=app.execute(action=do_cache, respawn=app.RespawnType.OPTIONAL),
+ description='Strip build cache and old build directories',
+ opts=[core.common_opts.ShowHelpOptions(), CollectCacheOptions(), CustomBuildRootOptions()],
+ visible=True,
+ )
+ self['dist_cache'] = core.yarg.OptsHandler(
+ action=app.execute(action=do_strip_yt_cache, respawn=app.RespawnType.NONE),
+ description='Strip distributed (YT) cache',
+ opts=[core.common_opts.ShowHelpOptions(), DistCacheSetupOptions()],
+ visible=False,
+ )
+
+
+class FilterBySize(object):
+ def __init__(self, size_limit):
+ self.size_limit = size_limit
+ self.total_size = 0
+ self._items = {}
+
+ def __call__(self, item):
+ if item.uid in self._items:
+ return False
+ size = item.size
+ self.total_size += size
+ self._items[item.uid] = item
+ return self.total_size < self.size_limit
+
+
+class FilterByObjectSize(object):
+ def __init__(self, size_limit):
+ self.size_limit = size_limit
+
+ def __call__(self, item):
+ return item.size < self.size_limit
+
+
+class FilterByAge(object):
+ def __init__(self, age_limit):
+ self.age_limit = age_limit
+ self.now = time.time()
+
+ def __call__(self, item):
+ return item.timestamp > self.now - self.age_limit
+
+
+def _clean_dir_selectively(logs, file_dir, file_name):
+ try:
+ for i in os.listdir(logs):
+ date_dir = os.path.join(logs, i)
+ if date_dir != file_dir:
+ fs.remove_tree_safe(date_dir)
+
+ if file_dir:
+ for i in os.listdir(file_dir):
+ if i != file_name:
+ fs.remove_tree_safe(os.path.join(file_dir, i))
+ except OSError as e:
+ logger.debug('Cleaning %s root failed %s', logs, e)
+
+
+def _clean_logs():
+ logger.debug('Cleaning logs root')
+
+ try:
+ import app_ctx
+
+ file_dir, file_name = os.path.split(app_ctx.file_log)
+ except Exception as e:
+ file_dir, file_name = ('', '')
+ logger.debug('Log name was not obtained %s', e)
+
+ logs = os.path.join(cc.misc_root(), 'logs')
+ _clean_dir_selectively(logs, file_dir, file_name)
+
+
+def _clean_evlogs():
+ logger.debug('Cleaning evlogs root')
+
+ try:
+ import app_ctx
+
+ file_dir, file_name = os.path.split(app_ctx.evlog._fileobj.name)
+ except Exception as e:
+ file_dir, file_name = ('', '')
+ logger.debug('Log name was not obtained %s', e)
+
+ evlogs = os.path.join(cc.misc_root(), 'evlogs')
+ _clean_dir_selectively(evlogs, file_dir, file_name)
+
+
+def _clean_tools():
+ running_ya_bin_dir = os.path.dirname(sys.argv[0])
+ base_running_ya_bin_dir = os.path.basename(running_ya_bin_dir).replace('_d', '')
+ errors = 0
+ for i in os.listdir(os.path.dirname(cc.tool_root())):
+ full_path = os.path.join(os.path.dirname(cc.tool_root()), i)
+ if i == 'v4':
+ if len(glob.glob(os.path.join(full_path, '*.corrupted'))) == 0:
+ tc.unlock_all()
+ continue
+ else:
+ errors += 1
+ elif full_path == running_ya_bin_dir:
+ continue
+ elif os.path.isfile(full_path):
+ try:
+ if os.path.getsize(full_path) < 1024 * 100:
+ with open(full_path, 'r') as f:
+ if base_running_ya_bin_dir in f.read():
+ continue
+ except Exception:
+ errors += 1
+ logger.debug('Cleaning tool %s', full_path)
+ fs.remove_tree_safe(full_path)
+ return errors
+
+
+def do_cache(opts):
+ build_root = opts.custom_build_directory or cc.build_root()
+
+ cache = None
+ try:
+ cache = ym.make_cache(opts, build_root)
+ except Exception:
+ logger.exception("While initializing cache")
+
+ lock = ym.make_lock(opts, build_root, write_lock=True)
+
+ with lock:
+ if opts.cache_stat:
+ import app_ctx
+
+ if not cache:
+ app_ctx.display.emit_message("Cache not initialized, can't show stats")
+ else:
+ cache.analyze(app_ctx.display)
+ errors = 0
+ else:
+ errors = _do_collect_cache(cache, build_root, opts)
+
+ if cache:
+ cache.flush()
+
+ logger.debug(
+ "ya gc stats %s",
+ {
+ 'cache_size': opts.cache_size,
+ 'object_size_limit': opts.object_size_limit,
+ 'age_limit': opts.age_limit,
+ 'symlinks_ttl': opts.symlinks_ttl,
+ 'errors': errors,
+ },
+ )
+
+
+def _do_collect_cache(cache, build_root, opts):
+ logger.debug('Cleaning tmp root')
+ fs.remove_tree_safe(cc.tmp_path())
+
+ logger.debug('Cleaning snowden root')
+ fs.remove_tree_safe(os.path.join(cc.misc_root(), 'snowden'))
+
+ logger.debug('Cleaning build root')
+ fs.remove_tree_safe(os.path.join(build_root, 'build_root'))
+
+ logger.debug('Cleaning conf root')
+ fs.remove_tree_safe(os.path.join(build_root, 'conf'))
+
+ errors = _clean_tools()
+
+ logger.debug('Cleaning tmp root')
+ fs.remove_tree_safe(os.path.join(cc.misc_root(), 'tmp'))
+
+ _clean_logs()
+ _clean_evlogs()
+
+ if not on_win():
+ logger.debug('Cleaning symres')
+ symres_dir = os.path.join(build_root, 'symres')
+ src_dir = cc.find_root(fail_on_error=False) or ""
+ if os.path.isdir(symres_dir):
+ logger.debug('Cleaning symres %s for %s, ttl=%s', symres_dir, src_dir, opts.symlinks_ttl)
+ result_store.SymlinkResultStore(symres_dir, src_dir).sieve(
+ state=None, ttl=opts.symlinks_ttl, cleanup=opts.symlinks_ttl == 0
+ )
+
+ if hasattr(cache, '_store_path'):
+ for dir in os.listdir(os.path.join(build_root, 'cache')):
+ full_path = os.path.join(build_root, 'cache', dir)
+ if cache._store_path == full_path:
+ if len(glob.glob(os.path.join(full_path, '*.corrupted'))) == 0:
+ continue
+ else:
+ errors += 1
+ logger.debug('Cleaning cache directory %s', full_path)
+ fs.remove_tree_safe(full_path)
+
+ if opts.cache_size is not None and opts.object_size_limit is None and opts.age_limit is None:
+ logger.debug('Cleaning for total size %s', opts.cache_size)
+ if hasattr(cache, 'strip'):
+ cache.strip(FilterBySize(opts.cache_size))
+ elif hasattr(cache, 'strip_total_size'):
+ cache.strip_total_size(opts.cache_size)
+
+ from yalibrary.toolscache import tc_force_gc
+
+ tc_force_gc(opts.cache_size)
+
+ if opts.object_size_limit is not None:
+ logger.debug('Cleaning for object size %s', opts.object_size_limit)
+ if hasattr(cache, 'strip'):
+ cache.strip(FilterByObjectSize(opts.object_size_limit))
+ elif hasattr(cache, 'strip_max_object_size'):
+ cache.strip_max_object_size(opts.object_size_limit)
+
+ if opts.age_limit is not None:
+ logger.debug('Cleaning for age %s', opts.age_limit)
+ if hasattr(cache, 'strip'):
+ cache.strip(FilterByAge(opts.age_limit))
+ elif hasattr(cache, 'strip_max_age'):
+ cache.strip_max_age(opts.age_limit)
+
+ return errors
+
+
+def do_strip_yt_cache(opts):
+ try:
+ from yalibrary.store.yt_store import yt_store
+ except ImportError as e:
+ logger.warn("YT store is not available: %s", e)
+
+ token = opts.yt_token or opts.oauth_token
+ cache = yt_store.YtStore(
+ opts.yt_proxy,
+ opts.yt_dir,
+ None,
+ token=token,
+ readonly=opts.yt_readonly,
+ max_cache_size=opts.yt_max_cache_size,
+ ttl=opts.yt_store_ttl,
+ )
+
+ counters = cache.strip()
+ if counters:
+ logger.info(
+ 'Deleted: meta rows:%d, data rows:%d, net data size:%d',
+ counters['meta_rows'],
+ counters['data_rows'],
+ counters['data_size'],
+ )
diff --git a/devtools/ya/handlers/gc/ya.make b/devtools/ya/handlers/gc/ya.make
new file mode 100644
index 0000000000..1b8296d1ee
--- /dev/null
+++ b/devtools/ya/handlers/gc/ya.make
@@ -0,0 +1,37 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.gc
+ __init__.py
+)
+
+PEERDIR(
+ contrib/python/six
+ devtools/ya/app
+ devtools/ya/build
+ devtools/ya/build/build_opts
+ devtools/ya/core
+ devtools/ya/core/config
+ devtools/ya/core/yarg
+ devtools/ya/exts
+ devtools/ya/yalibrary/runner
+ devtools/ya/yalibrary/store
+ devtools/ya/yalibrary/toolscache
+ library/python/fs
+ library/python/windows
+)
+
+IF (NOT YA_OPENSOURCE)
+ PEERDIR(
+ devtools/ya/yalibrary/store/yt_store # see YA-938
+ )
+ENDIF()
+
+END()
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/gen_config/__init__.py b/devtools/ya/handlers/gen_config/__init__.py
new file mode 100644
index 0000000000..e771dac871
--- /dev/null
+++ b/devtools/ya/handlers/gen_config/__init__.py
@@ -0,0 +1,48 @@
+from __future__ import absolute_import
+import core.common_opts
+import core.yarg
+
+from . import gen_config
+
+
+class GenConfigOptions(core.yarg.Options):
+ def __init__(self):
+ self.output = None
+ self.dump_defaults = False
+
+ @staticmethod
+ def consumer():
+ return [
+ core.yarg.SingleFreeArgConsumer(
+ help='ya.conf',
+ hook=core.yarg.SetValueHook('output'),
+ required=False,
+ ),
+ core.yarg.ArgConsumer(
+ ['--dump-defaults'],
+ help='Dump default values as JSON',
+ hook=core.yarg.SetConstValueHook('dump_defaults', True),
+ ),
+ ]
+
+
+class GenConfigYaHandler(core.yarg.OptsHandler):
+ description = 'Generate default ya config'
+
+ def __init__(self):
+ self._root_handler = None
+ super(GenConfigYaHandler, self).__init__(
+ action=self.do_generate,
+ description=self.description,
+ opts=[
+ core.common_opts.ShowHelpOptions(),
+ GenConfigOptions(),
+ ],
+ )
+
+ def handle(self, root_handler, args, prefix):
+ self._root_handler = root_handler
+ super(GenConfigYaHandler, self).handle(root_handler, args, prefix)
+
+ def do_generate(self, args):
+ gen_config.generate_config(self._root_handler, args.output, args.dump_defaults)
diff --git a/devtools/ya/handlers/gen_config/gen_config.py b/devtools/ya/handlers/gen_config/gen_config.py
new file mode 100644
index 0000000000..56a0d6fc64
--- /dev/null
+++ b/devtools/ya/handlers/gen_config/gen_config.py
@@ -0,0 +1,282 @@
+from __future__ import absolute_import
+import sys
+import itertools
+import collections
+import json
+
+import toml
+
+import core.yarg
+import six
+
+toml_encoder = toml.TomlEncoder()
+
+NOT_SET = -1
+
+
+def is_jsonable(x):
+ try:
+ json.dumps(x)
+ return True
+ except (TypeError, OverflowError):
+ return False
+
+
+def iter_ya_options(root_handler, all_opts=False):
+ # prefer setvalue over setconstvalue to provide correct description and default value
+ hook_prefer_order = [core.yarg.SetValueHook, core.yarg.SetConstValueHook]
+
+ def get_hook_priority(hook, consumer):
+ if isinstance(consumer, core.yarg.ConfigConsumer):
+ return -1
+
+ htype = type(hook)
+ if htype in hook_prefer_order:
+ pos = hook_prefer_order.index(htype)
+ else:
+ pos = len(hook_prefer_order)
+ return len(hook_prefer_order) - pos
+
+ for handler_name, command in six.iteritems(root_handler.sub_handlers):
+ if not command.visible:
+ continue
+
+ top_handler = command.command()
+ handlers = collections.deque([([handler_name], top_handler)])
+
+ while handlers:
+ trace, handler = handlers.pop()
+
+ sub_handlers = handler.sub_handlers or {}
+ for sub_name in sorted(sub_handlers):
+ handlers.append((trace + [sub_name], sub_handlers[sub_name]))
+
+ options = getattr(handler, "options", None)
+ if not options:
+ continue
+
+ if all_opts:
+ for opt in options:
+ for name in vars(opt).keys():
+ default = getattr(opt, name)
+ if is_jsonable(default):
+ yield name, "", default, "", "", ""
+ continue
+
+ top_consumer = options.consumer()
+ if isinstance(top_consumer, core.yarg.Compound):
+ consumers = top_consumer.parts
+ else:
+ consumers = [top_consumer]
+
+ # pairwise config option with command line name if it's possible
+ params = {}
+ for consumer in consumers:
+ if not isinstance(consumer, (core.yarg.ArgConsumer, core.yarg.ConfigConsumer)):
+ continue
+
+ hook = consumer.hook
+ if not hook:
+ continue
+
+ opt_name = getattr(hook, 'name', None)
+ if not opt_name or opt_name.startswith("_"):
+ continue
+
+ if opt_name not in params:
+ params[opt_name] = {
+ 'configurable': False,
+ }
+
+ entry = params[opt_name]
+ priority = get_hook_priority(hook, consumer)
+
+ entry['configurable'] |= isinstance(consumer, core.yarg.ConfigConsumer)
+ if priority > entry.get('_priority', -2):
+ entry['cmdline_names'] = get_arg_consumer_names(consumer) or entry.get('cmdline_names')
+ entry['trace'] = trace or entry.get('trace')
+ entry['description'] = getattr(consumer, "help", "") or entry.get('description')
+ entry['group'] = getattr(consumer, "group", "") or entry.get('group', 0)
+ # default value might be empty list or dict
+ if getattr(options, opt_name, None) is None:
+ entry['default'] = entry.get('default')
+ else:
+ entry['default'] = getattr(options, opt_name)
+ entry['_priority'] = priority
+
+ for name, x in params.items():
+ if x['configurable']:
+ yield name, x['cmdline_names'], x['default'], x['group'], x['description'], x['trace']
+
+
+def get_arg_consumer_names(consumer):
+ if isinstance(consumer, core.yarg.ArgConsumer):
+ return list([_f for _f in [consumer.short_name, consumer.long_name] if _f])
+ return []
+
+
+def split(iterable, func):
+ d1 = []
+ d2 = []
+ for entry in iterable:
+ if func(entry):
+ d1.append(entry)
+ else:
+ d2.append(entry)
+ return d1, d2
+
+
+def get_comment(entry):
+ parts = []
+ if entry['desc']:
+ parts.append(entry['desc'])
+ if entry['cmdline_names']:
+ parts.append("({})".format(", ".join(entry['cmdline_names'])))
+ return " ".join(parts)
+
+
+def compress_keys(data, result, trace=None):
+ trace = trace or []
+ for key, val in six.iteritems(data):
+ if isinstance(val, dict):
+ result[".".join(trace + [key])] = val
+ compress_keys(val, result, trace + [key])
+
+
+def dump_options(entries, output, config, section="", print_comment=True):
+ if section:
+ output.append("[{}]".format(section))
+ for entry in sorted(entries, key=lambda x: x['name']):
+ parts = []
+ if print_comment:
+ comment = get_comment(entry)
+ if comment:
+ parts.append("# {}".format(comment))
+ name = entry['name']
+ if name in config and config[name] != entry['default']:
+ parts.append("{} = {}".format(name, toml_encoder.dump_value(config[name])))
+ else:
+ parts.append("# {} = {}".format(name, toml_encoder.dump_value(entry['default'])))
+
+ output.append("\n".join(parts))
+
+
+def dump_subgroups(subgroups, output, config, section="", print_comment=True):
+ def get_section(name):
+ return ".".join([_f for _f in [section, name] if _f])
+
+ for entry in subgroups:
+ parts = []
+ if print_comment:
+ comment = get_comment(entry)
+ if comment:
+ parts.append("# {}".format(comment))
+
+ name = entry['name']
+ if name in config and config[name] != entry['default']:
+ parts.append("[{}]".format(get_section(name)))
+ for key, val in config[name].items():
+ parts.append("{} = {}".format(key, toml_encoder.dump_value(val)))
+ else:
+ parts.append("# [{}]".format(get_section(name)))
+ for key, val in entry['default'].items():
+ parts.append("# {} = {}".format(key, toml_encoder.dump_value(val)))
+
+ output.append("\n".join(parts))
+
+
+def dump_config(options, handler_map, user_config):
+ def sort_func(x):
+ return getattr(x['group'], 'index', NOT_SET)
+
+ def get_group_title(name):
+ return "{} {} {}".format("=" * 10, name, "=" * (65 - len(name)))
+
+ blocks = [
+ "# Save config to the junk/{USER}/ya.conf or to the ~/.ya/ya.conf\n# For more info see https://docs.yandex-team.ru/yatool/commands/gen_config"
+ ]
+
+ # dump all options
+ subgroups = []
+ data = sorted(options.values(), key=sort_func)
+ for group_index, entries in itertools.groupby(data, sort_func):
+ entries = list(entries)
+
+ if group_index != NOT_SET:
+ group_name = entries[0]['group'].name
+ if group_name:
+ blocks.append("# " + get_group_title(group_name))
+
+ subs, opts = split(entries, lambda x: isinstance(x['default'], dict))
+ dump_options(opts, blocks, user_config)
+ # move all subgroup options out of the default section
+ # otherwise every option defined after section will be related to the section
+ subgroups += subs
+
+ if subgroups:
+ blocks.append("# " + get_group_title("Various table options"))
+ blocks.append("# Uncomment table name with parameters")
+ dump_subgroups(subgroups, blocks, user_config)
+
+ # Save user redefined opts for specific handlers
+ specific_opts = {}
+ user_handler_map = {}
+ compress_keys(user_config, user_handler_map)
+ for section, keys in six.iteritems(user_handler_map):
+ if section not in handler_map:
+ continue
+ entries = []
+ for optname in keys:
+ if optname in handler_map[section]:
+ entries.append(options[optname])
+ if entries:
+ specific_opts[section] = entries
+
+ if specific_opts:
+ blocks.append("# " + get_group_title("Redefined options for specific handlers"))
+ for section, entries in sorted(specific_opts.items(), key=lambda x: x[0]):
+ subs, opts = split(entries, lambda x: isinstance(x['default'], dict))
+ if opts:
+ dump_options(opts, blocks, user_handler_map[section], section, print_comment=False)
+ if subgroups:
+ dump_subgroups(subs, blocks, user_handler_map[section], section, print_comment=False)
+
+ return "\n#\n".join(blocks)
+
+
+def generate_config(root_handler, output=None, dump_defaults=None):
+ # Don't load global config files to avoid penetration of the global options into user config
+ config_files = core.yarg.get_config_files(global_config=False)
+ user_config = core.yarg.load_config(config_files)
+
+ options = {}
+ handler_map = {}
+ for name, cmdline_names, default, group, desc, trace in iter_ya_options(root_handler, dump_defaults):
+ if name not in options:
+ options[name] = {
+ 'cmdline_names': cmdline_names,
+ 'default': default,
+ 'desc': desc,
+ 'group': group,
+ 'name': name,
+ }
+ else:
+ entry = options[name]
+ entry['desc'] = entry['desc'] or desc
+ entry['group'] = entry['group'] or group
+
+ target = ".".join(trace)
+ if target not in handler_map:
+ handler_map[target] = []
+ handler_map[target].append(name)
+
+ if dump_defaults:
+ json.dump({k: v['default'] for k, v in options.items()}, sys.stdout, indent=2)
+ return
+
+ data = dump_config(options, handler_map, user_config)
+ if output:
+ with open(output, 'w') as afile:
+ afile.write(data)
+ else:
+ sys.stdout.write(data + '\n')
diff --git a/devtools/ya/handlers/gen_config/ya.make b/devtools/ya/handlers/gen_config/ya.make
new file mode 100644
index 0000000000..b5fdeece7c
--- /dev/null
+++ b/devtools/ya/handlers/gen_config/ya.make
@@ -0,0 +1,23 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.gen_config
+ __init__.py
+ gen_config.py
+)
+
+PEERDIR(
+ contrib/python/six
+ contrib/python/toml
+ devtools/ya/core
+ devtools/ya/core/yarg
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/ide/__init__.py b/devtools/ya/handlers/ide/__init__.py
new file mode 100644
index 0000000000..4d28d29a69
--- /dev/null
+++ b/devtools/ya/handlers/ide/__init__.py
@@ -0,0 +1,734 @@
+from __future__ import absolute_import
+import os.path
+
+import six
+
+import core.yarg
+import core.config
+import core.common_opts
+
+import build.build_opts
+import build.compilation_database as bcd
+
+import ide.ide_common
+import ide.msvs
+import ide.msvs_lite
+import ide.clion2016
+import ide.idea
+import ide.qt
+import ide.remote_ide_qt
+import ide.goland
+import ide.pycharm
+import ide.venv
+import ide.vscode_all
+import ide.vscode_clangd
+import ide.vscode_go
+import ide.vscode_py
+import ide.vscode_ts
+import ide.vscode.opts
+
+import yalibrary.platform_matcher as pm
+
+import app
+import app_config
+
+if app_config.in_house:
+ import devtools.ya.ide.fsnotifier
+
+from core.yarg.help_level import HelpLevel
+
+if six.PY3:
+ import ide.gradle
+
+
+class TidyOptions(core.yarg.Options):
+ def __init__(self):
+ self.setup_tidy = False
+
+ @staticmethod
+ def consumer():
+ return [
+ core.yarg.ArgConsumer(
+ ['--setup-tidy'],
+ help="Setup default arcadia's clang-tidy config in a project",
+ hook=core.yarg.SetConstValueHook('setup_tidy', True),
+ group=core.yarg.BULLET_PROOF_OPT_GROUP,
+ ),
+ core.yarg.ConfigConsumer('setup_tidy'),
+ ]
+
+
+class CLionOptions(core.yarg.Options):
+ CLION_OPT_GROUP = core.yarg.Group('CLion project options', 0)
+
+ def __init__(self):
+ self.filters = []
+ self.lite_mode = False
+ self.remote_toolchain = None
+ self.remote_deploy_config = None
+ self.remote_repo_path = None
+ self.remote_build_path = None
+ self.remote_deploy_host = None
+ self.use_sync_server = False
+ self.content_root = None
+ self.strip_non_final_targets = False
+ self.full_targets = False
+ self.add_py_targets = False
+
+ @staticmethod
+ def consumer():
+ return [
+ core.yarg.ArgConsumer(
+ ['--filter', '-f'],
+ help='Only consider filtered content',
+ hook=core.yarg.SetAppendHook('filters'),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--mini', '-m'],
+ help='Lite mode for solution (fast open, without build)',
+ hook=core.yarg.SetConstValueHook('lite_mode', True),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--remote-toolchain'],
+ help='Generate configurations for remote toolchain with this name',
+ hook=core.yarg.SetValueHook('remote_toolchain'),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--remote-deploy-config'],
+ help='Name of the remote server configuration tied to the remote toolchain',
+ hook=core.yarg.SetValueHook('remote_deploy_config'),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--remote-repo-path'],
+ help='Path to the arc repository at the remote host',
+ hook=core.yarg.SetValueHook('remote_repo_path'),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--remote-build-path'],
+ help='Path to the directory for CMake output at the remote host',
+ hook=core.yarg.SetValueHook('remote_build_path'),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--remote-host'],
+ help='Hostname associated with remote server configuration',
+ hook=core.yarg.SetValueHook('remote_deploy_host'),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--use-sync-server'],
+ help='Deploy local files via sync server instead of file watchers',
+ hook=core.yarg.SetConstValueHook('use_sync_server', True),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--project-root', '-r'],
+ help='Root directory for a CLion project',
+ hook=core.yarg.SetValueHook('content_root'),
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--strip-non-final-targets'],
+ hook=core.yarg.SetConstValueHook('strip_non_final_targets', True),
+ help='Do not create target for non-final nodes',
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--full-targets'],
+ hook=core.yarg.SetConstValueHook('full_targets', True),
+ help='Old Mode: Enable full targets graph generation for project.',
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--add-py3-targets'],
+ hook=core.yarg.SetConstValueHook('add_py_targets', True),
+ help='Add Python 3 targets to project',
+ group=CLionOptions.CLION_OPT_GROUP,
+ ),
+ core.yarg.ConfigConsumer('filters'),
+ core.yarg.ConfigConsumer('lite_mode'),
+ core.yarg.ConfigConsumer('remote_toolchain'),
+ core.yarg.ConfigConsumer('remote_deploy_config'),
+ core.yarg.ConfigConsumer('remote_repo_path'),
+ core.yarg.ConfigConsumer('remote_build_path'),
+ core.yarg.ConfigConsumer('remote_deploy_host'),
+ core.yarg.ConfigConsumer('use_sync_server'),
+ core.yarg.ConfigConsumer('content_root'),
+ core.yarg.ConfigConsumer('strip_non_executable_target'),
+ core.yarg.ConfigConsumer('full_targets'),
+ core.yarg.ConfigConsumer('add_py_targets'),
+ ]
+
+ def postprocess2(self, params):
+ if ' ' in params.project_title:
+ raise core.yarg.ArgsValidatingException('Clion project title should not contain space symbol')
+ if params.add_py_targets and params.full_targets:
+ raise core.yarg.ArgsValidatingException('--add-py-targets must be used without --full-targets')
+
+
+class IdeaOptions(core.yarg.Options):
+ IDEA_OPT_GROUP = core.yarg.Group('Idea project options', 0)
+ IDE_PLUGIN_INTEGRATION_GROUP = core.yarg.Group('Integration wuth IDE plugin', 1)
+
+ def __init__(self):
+ self.idea_project_root = None
+ self.local = False
+ self.group_modules = None
+ self.dry_run = False
+ self.ymake_bin = None
+ self.iml_in_project_root = False
+ self.iml_keep_relative_paths = False
+ self.idea_files_root = None
+ self.project_name = None
+ self.minimal = False
+ self.directory_based = True
+ self.omit_test_data = False
+ self.with_content_root_modules = False
+ self.external_content_root_modules = []
+ self.generate_tests_run = False
+ self.generate_tests_for_deps = False
+ self.separate_tests_modules = False
+ self.auto_exclude_symlinks = False
+ self.exclude_dirs = []
+ self.with_common_jvm_args_in_junit_template = False
+ self.with_long_library_names = False
+ self.copy_shared_index_config = False
+ self.idea_jdk_version = None
+ self.regenerate_with_project_update = False
+ self.project_update_targets = []
+ self.project_update_kind = None
+
+ @staticmethod
+ def consumer():
+ return [
+ core.yarg.ArgConsumer(
+ ['-r', '--project-root'],
+ help='IntelliJ IDEA project root path',
+ hook=core.yarg.SetValueHook('idea_project_root'),
+ group=IdeaOptions.IDEA_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['-P', '--project-output'],
+ help='IntelliJ IDEA project root path. Please, use instead of -r',
+ hook=core.yarg.SetValueHook('idea_project_root'),
+ group=IdeaOptions.IDEA_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['-l', '--local'],
+ help='Only recurse reachable projects are idea modules',
+ hook=core.yarg.SetConstValueHook('local', True),
+ group=IdeaOptions.IDEA_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--group-modules'],
+ help='Group idea modules according to paths: (tree, flat)',
+ hook=core.yarg.SetValueHook('group_modules', values=('tree', 'flat')),
+ group=IdeaOptions.IDEA_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['-n', '--dry-run'],
+ help='Emulate create project, but do nothing',
+ hook=core.yarg.SetConstValueHook('dry_run', True),
+ group=IdeaOptions.IDEA_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--ymake-bin'], help='Path to ymake binary', hook=core.yarg.SetValueHook('ymake_bin'), visible=False
+ ),
+ core.yarg.ArgConsumer(
+ ['--iml-in-project-root'],
+ help='Store ".iml" files in project root tree(stores in source root tree by default)',
+ hook=core.yarg.SetConstValueHook('iml_in_project_root', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--iml-keep-relative-paths'],
+ help='Keep relative paths in ".iml" files (works with --iml-in-project-root)',
+ hook=core.yarg.SetConstValueHook('iml_keep_relative_paths', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--idea-files-root'],
+ help='Root for .ipr and .iws files',
+ hook=core.yarg.SetValueHook('idea_files_root'),
+ ),
+ core.yarg.ArgConsumer(
+ ['--project-name'],
+ help='Idea project name (.ipr and .iws file)',
+ hook=core.yarg.SetValueHook('project_name'),
+ ),
+ core.yarg.ArgConsumer(
+ ['--ascetic'],
+ help='Create the minimum set of project settings',
+ hook=core.yarg.SetConstValueHook('minimal', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--directory-based'],
+ help='Create project in actual (directory based) format',
+ hook=core.yarg.SetConstValueHook('directory_based', True),
+ visible=False,
+ ),
+ core.yarg.ArgConsumer(
+ ['--omit-test-data'],
+ help='Do not export test_data',
+ hook=core.yarg.SetConstValueHook('omit_test_data', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--with-content-root-modules'],
+ help='Generate content root modules',
+ hook=core.yarg.SetConstValueHook('with_content_root_modules', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--external-content-root-module'],
+ help='Add external content root modules',
+ hook=core.yarg.SetAppendHook('external_content_root_modules'),
+ ),
+ core.yarg.ArgConsumer(
+ ['--generate-junit-run-configurations'],
+ help='Generate run configuration for junit tests',
+ hook=core.yarg.SetConstValueHook('generate_tests_run', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--generate-tests-for-dependencies'],
+ help='Generate tests for PEERDIR dependencies',
+ hook=core.yarg.SetConstValueHook('generate_tests_for_deps', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--separate-tests-modules'],
+ help='Do not merge tests modules with their own libraries',
+ hook=core.yarg.SetConstValueHook('separate_tests_modules', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--auto-exclude-symlinks'],
+ help='Add all symlink-dirs in modules to exclude dirs',
+ hook=core.yarg.SetConstValueHook('auto_exclude_symlinks', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--exclude-dirs'],
+ help='Exclude dirs with specific names from all modules',
+ hook=core.yarg.SetAppendHook('exclude_dirs'),
+ ),
+ core.yarg.ArgConsumer(
+ ['--with-common-jvm-args-in-junit-template'],
+ help='Add common JVM_ARGS flags to default junit template',
+ hook=core.yarg.SetConstValueHook('with_common_jvm_args_in_junit_template', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--with-long-library-names'],
+ help='Generate long library names',
+ hook=core.yarg.SetConstValueHook('with_long_library_names', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--copy-shared-index-config'],
+ help='Copy project config for Shared Indexes if exist',
+ hook=core.yarg.SetConstValueHook('copy_shared_index_config', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--idea-jdk-version'],
+ help='Project JDK version',
+ hook=core.yarg.SetValueHook('idea_jdk_version'),
+ ),
+ core.yarg.ArgConsumer(
+ ['-U', '--regenerate-with-project-update'],
+ help='Run `ya project update` upon regeneration from Idea',
+ group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP,
+ hook=core.yarg.SetConstValueHook('regenerate_with_project_update', True),
+ ),
+ core.yarg.ArgConsumer(
+ ['--project-update-targets'],
+ help='Run `ya project update` for this dirs upon regeneration from Idea',
+ hook=core.yarg.SetAppendHook('project_update_targets'),
+ group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP,
+ visible=HelpLevel.ADVANCED,
+ ),
+ core.yarg.ArgConsumer(
+ ['--project-update-kind'],
+ help='Type of a project to use in `ya project update` upon regernation from Idea',
+ hook=core.yarg.SetValueHook('project_update_kind'),
+ group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP,
+ visible=HelpLevel.ADVANCED,
+ ),
+ core.yarg.ConfigConsumer('idea_project_root'),
+ core.yarg.ConfigConsumer('local'),
+ core.yarg.ConfigConsumer('group_modules'),
+ core.yarg.ConfigConsumer('dry_run'),
+ core.yarg.ConfigConsumer('iml_in_project_root'),
+ core.yarg.ConfigConsumer('iml_keep_relative_paths'),
+ core.yarg.ConfigConsumer('idea_files_root'),
+ core.yarg.ConfigConsumer('project_name'),
+ core.yarg.ConfigConsumer('minimal'),
+ core.yarg.ConfigConsumer('directory_based'),
+ core.yarg.ConfigConsumer('omit_test_data'),
+ core.yarg.ConfigConsumer('with_content_root_modules'),
+ core.yarg.ConfigConsumer('external_content_root_modules'),
+ core.yarg.ConfigConsumer('generate_tests_run'),
+ core.yarg.ConfigConsumer('generate_tests_for_deps'),
+ core.yarg.ConfigConsumer('separate_tests_modules'),
+ core.yarg.ConfigConsumer('auto_exclude_symlinks'),
+ core.yarg.ConfigConsumer('exclude_dirs'),
+ core.yarg.ConfigConsumer('with_common_jvm_args_in_junit_template'),
+ core.yarg.ConfigConsumer('with_long_library_names'),
+ core.yarg.ConfigConsumer('copy_shared_index_config'),
+ core.yarg.ConfigConsumer('idea_jdk_version'),
+ core.yarg.ConfigConsumer('regenarate_with_project_update'),
+ core.yarg.ConfigConsumer('project_update_targets'),
+ core.yarg.ConfigConsumer('project_update_kind'),
+ ]
+
+ def postprocess(self):
+ if self.idea_project_root is None:
+ raise core.yarg.ArgsValidatingException('Idea project root(-r, --project-root) must be specified.')
+
+ self.idea_project_root = os.path.abspath(self.idea_project_root)
+
+ if self.iml_keep_relative_paths and not self.iml_in_project_root:
+ raise core.yarg.ArgsValidatingException(
+ '--iml-keep-relative-paths can be used only with --iml-in-project-root'
+ )
+
+ if self.generate_tests_run and not self.directory_based:
+ raise core.yarg.ArgsValidatingException(
+ 'run configurations may be generated only for directory-based project'
+ )
+
+ for p in self.exclude_dirs:
+ if os.path.isabs(p):
+ raise core.yarg.ArgsValidatingException('Absolute paths are not allowed in --exclude-dirs')
+
+
+class GradleOptions(core.yarg.Options):
+ GRADLE_OPT_GROUP = core.yarg.Group('Gradle project options', 0)
+
+ def __init__(self):
+ self.gradle_name = None
+
+ @staticmethod
+ def consumer():
+ return [
+ core.yarg.ArgConsumer(
+ ['--gradle-name'],
+ help='Set project name manually',
+ hook=core.yarg.SetValueHook('gradle_name'),
+ group=GradleOptions.GRADLE_OPT_GROUP,
+ )
+ ]
+
+
+class PycharmOptions(core.yarg.Options):
+ PYCHARM_OPT_GROUP = core.yarg.Group('Pycharm project options', 0)
+ PYTHON_WRAPPER_NAME = 'pycharm_python_wrapper'
+
+ def __init__(self):
+ self.only_generate_wrapper = False
+ self.wrapper_name = PycharmOptions.PYTHON_WRAPPER_NAME
+ self.list_ide = False
+ self.ide_version = None
+
+ @staticmethod
+ def consumer():
+ return [
+ core.yarg.ArgConsumer(
+ ['--only-generate-wrapper'],
+ help="Don not generate Pycharm project, only wrappers",
+ hook=core.yarg.SetConstValueHook('only_generate_wrapper', True),
+ group=PycharmOptions.PYCHARM_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--wrapper-name'],
+ help='Name of generated python wrapper. Use `python` for manual adding wrapper as Python SDK.',
+ hook=core.yarg.SetValueHook('wrapper_name'),
+ group=PycharmOptions.PYCHARM_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--list-ide'],
+ help='List available JB IDE for patching SDK list.',
+ hook=core.yarg.SetConstValueHook('list_ide', True),
+ group=PycharmOptions.PYCHARM_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--ide-version'],
+ help='Change IDE version for patching SDK list. Available IDE: {}'.format(
+ ", ".join(ide.pycharm.find_available_ide())
+ ),
+ hook=core.yarg.SetValueHook('ide_version'),
+ group=PycharmOptions.PYCHARM_OPT_GROUP,
+ ),
+ ]
+
+ def postprocess(self):
+ if PycharmOptions.PYTHON_WRAPPER_NAME != self.wrapper_name and not self.only_generate_wrapper:
+ raise core.yarg.ArgsValidatingException(
+ "Custom wrapper name can be used with option --only-generate-wrapper"
+ )
+
+
+MSVS_OPTS = ide.msvs.MSVS_OPTS + [ide.ide_common.YaExtraArgsOptions(), core.common_opts.YaBin3Options()]
+
+
+def gen_msvs_solution(params):
+ impl = ide.msvs_lite if params.lite else ide.msvs
+ return impl.gen_msvs_solution(params)
+
+
+def get_description(text, ref_name):
+ if app_config.in_house:
+ ref = {
+ "c": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#c",
+ "golang": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#golang",
+ "multi": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#multi",
+ "python": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#python",
+ "typescript": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#typescript",
+ }[ref_name]
+ return "{}\nDocs: [[c:dark-cyan]]{}[[rst]]".format(text, ref)
+ else:
+ return text
+
+
+class IdeYaHandler(core.yarg.CompositeHandler):
+ description = 'Generate project for IDE'
+
+ def __init__(self):
+ core.yarg.CompositeHandler.__init__(self, description=self.description)
+ self['msvs'] = core.yarg.OptsHandler(
+ action=app.execute(gen_msvs_solution),
+ description='[[imp]]ya ide msvs[[rst]] is deprecated, please use clangd-based tooling instead',
+ opts=MSVS_OPTS,
+ examples=[
+ core.yarg.UsageExample(
+ '{prefix} util/generic util/datetime',
+ 'Generate solution for util/generic, util/datetime and all their dependencies',
+ ),
+ core.yarg.UsageExample('{prefix} -P Output', 'Generate solution in Output directory'),
+ core.yarg.UsageExample('{prefix} -T my_solution', 'Generate solution titled my_solution.sln'),
+ ],
+ visible=(pm.my_platform() == 'win32'),
+ )
+ self['clion'] = core.yarg.OptsHandler(
+ action=app.execute(ide.clion2016.do_clion),
+ description='[[imp]]ya ide clion[[rst]] is deprecated, please use clangd-based tooling instead',
+ opts=ide.ide_common.ide_via_ya_make_opts()
+ + [
+ CLionOptions(),
+ TidyOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ )
+
+ self['idea'] = core.yarg.OptsHandler(
+ action=app.execute(ide.idea.do_idea),
+ description='Generate stub for IntelliJ IDEA',
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ ide.ide_common.IdeYaMakeOptions(),
+ ide.ide_common.YaExtraArgsOptions(),
+ IdeaOptions(),
+ core.common_opts.OutputStyleOptions(),
+ core.common_opts.CrossCompilationOptions(),
+ core.common_opts.PrintStatisticsOptions(),
+ build.build_opts.ContinueOnFailOptions(),
+ build.build_opts.YMakeDebugOptions(),
+ build.build_opts.BuildThreadsOptions(build_threads=None),
+ build.build_opts.DistCacheOptions(),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.IgnoreRecursesOptions(),
+ build.build_opts.ContentUidsOptions(),
+ build.build_opts.ExecutorOptions(),
+ build.build_opts.CustomFetcherOptions(),
+ build.build_opts.SandboxAuthOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ unknown_args_as_free=True,
+ )
+ ide_gradle_opts = ide.ide_common.ide_minimal_opts(targets_free=True) + [
+ ide.ide_common.YaExtraArgsOptions(),
+ GradleOptions(),
+ core.yarg.ShowHelpOptions(),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.CustomFetcherOptions(),
+ build.build_opts.SandboxAuthOptions(),
+ core.common_opts.CrossCompilationOptions(),
+ build.build_opts.ToolsOptions(),
+ build.build_opts.BuildTypeOptions('release'),
+ build.build_opts.JavaSpecificOptions(),
+ ]
+ if six.PY2:
+ self['gradle'] = core.yarg.OptsHandler(
+ action=app.execute(
+ lambda *a, **k: None,
+ handler_python_major_version=3,
+ ),
+ description='Generate gradle for project',
+ opts=ide_gradle_opts,
+ visible=False,
+ )
+ if six.PY3:
+ self['gradle'] = core.yarg.OptsHandler(
+ action=app.execute(
+ ide.gradle.do_gradle,
+ handler_python_major_version=3,
+ ),
+ description='Generate gradle for project',
+ opts=ide_gradle_opts,
+ visible=False,
+ )
+ self['qt'] = core.yarg.OptsHandler(
+ action=app.execute(self._choose_qt_handler),
+ description='[[imp]]ya ide qt[[rst]] is deprecated, please use clangd-based tooling instead',
+ opts=ide.qt.QT_OPTS + [core.common_opts.YaBin3Options()],
+ )
+ self['goland'] = core.yarg.OptsHandler(
+ action=app.execute(ide.goland.do_goland),
+ description='Generate stub for Goland',
+ opts=ide.ide_common.ide_via_ya_make_opts()
+ + [
+ ide.goland.GolandOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ )
+ self['pycharm'] = core.yarg.OptsHandler(
+ action=app.execute(ide.pycharm.do_pycharm),
+ description='Generate PyCharm project.',
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ PycharmOptions(),
+ ide.ide_common.IdeYaMakeOptions(),
+ ide.ide_common.YaExtraArgsOptions(),
+ build.build_opts.DistCacheOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ visible=(pm.my_platform() != 'win32'),
+ )
+ self['vscode-clangd'] = core.yarg.OptsHandler(
+ action=app.execute(ide.vscode_clangd.gen_vscode_workspace),
+ description=get_description('Generate VSCode clangd C++ project.', ref_name='c'),
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ ide.vscode_clangd.VSCodeClangdOptions(),
+ ide.ide_common.YaExtraArgsOptions(),
+ bcd.CompilationDatabaseOptions(),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.OutputOptions(),
+ build.build_opts.BuildThreadsOptions(build_threads=None),
+ build.build_opts.ArcPrefetchOptions(),
+ build.build_opts.ContentUidsOptions(),
+ build.build_opts.ToolsOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ visible=(pm.my_platform() != 'win32'),
+ )
+ self['vscode-go'] = core.yarg.OptsHandler(
+ action=app.execute(ide.vscode_go.gen_vscode_workspace),
+ description=get_description('Generate VSCode Go project.', ref_name='golang'),
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ ide.vscode_go.VSCodeGoOptions(),
+ ide.ide_common.YaExtraArgsOptions(),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.BuildThreadsOptions(build_threads=None),
+ build.build_opts.ArcPrefetchOptions(),
+ build.build_opts.ContentUidsOptions(),
+ build.build_opts.ToolsOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ )
+ self['vscode-py'] = core.yarg.OptsHandler(
+ action=app.execute(ide.vscode_py.gen_vscode_workspace),
+ description=get_description('Generate VSCode Python project.', ref_name='python'),
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ ide.vscode_py.VSCodePyOptions(),
+ ide.ide_common.YaExtraArgsOptions(),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.BuildThreadsOptions(build_threads=None),
+ build.build_opts.ArcPrefetchOptions(),
+ build.build_opts.ContentUidsOptions(),
+ build.build_opts.ToolsOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ visible=(pm.my_platform() != 'win32'),
+ )
+ self['vscode-ts'] = core.yarg.OptsHandler(
+ action=app.execute(ide.vscode_py.gen_vscode_workspace),
+ description=get_description('Generate VSCode TypeScript project.', ref_name='typescript'),
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ ide.vscode_py.VSCodePyOptions(),
+ ide.ide_common.YaExtraArgsOptions(),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.BuildThreadsOptions(build_threads=None),
+ build.build_opts.ArcPrefetchOptions(),
+ build.build_opts.ContentUidsOptions(),
+ build.build_opts.ToolsOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ visible=(pm.my_platform() != 'win32'),
+ )
+ self['vscode'] = core.yarg.OptsHandler(
+ action=app.execute(ide.vscode_all.gen_vscode_workspace),
+ description=get_description('Generate VSCode multi-language project.', ref_name='multi'),
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ ide.vscode.opts.VSCodeAllOptions(),
+ ide.ide_common.YaExtraArgsOptions(),
+ bcd.CompilationDatabaseOptions(),
+ build.build_opts.ArcPrefetchOptions(prefetch=True, visible=False),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.OutputOptions(),
+ build.build_opts.BuildThreadsOptions(build_threads=None),
+ build.build_opts.ContentUidsOptions(),
+ build.build_opts.ToolsOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ )
+ self['vscode-ts'] = core.yarg.OptsHandler(
+ action=app.execute(ide.vscode_ts.gen_vscode_workspace),
+ description=get_description('Generate VSCode TypeScript project.', ref_name='typescript'),
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ ide.vscode_ts.VSCodeTypeScriptOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ visible=False, # FIXME: remove when ready for public release
+ )
+ self['venv'] = core.yarg.OptsHandler(
+ action=app.execute(ide.venv.do_venv),
+ description='Create or update python venv',
+ opts=ide.ide_common.ide_minimal_opts(targets_free=True)
+ + [
+ build.build_opts.BuildTypeOptions('release'),
+ build.build_opts.BuildThreadsOptions(build_threads=None),
+ build.build_opts.ExecutorOptions(),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.IgnoreRecursesOptions(),
+ build.build_opts.RebuildOptions(),
+ core.common_opts.BeVerboseOptions(),
+ core.common_opts.CrossCompilationOptions(),
+ ide.ide_common.YaExtraArgsOptions(),
+ ide.venv.VenvOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ visible=(pm.my_platform() != 'win32'),
+ )
+ if app_config.in_house:
+ self['fix-jb-fsnotifier'] = core.yarg.OptsHandler(
+ action=app.execute(devtools.ya.ide.fsnotifier.fix_fsnotifier),
+ description='Replace fsnotifier for JB IDEs.',
+ opts=[
+ devtools.ya.ide.fsnotifier.FixFsNotifierOptions(),
+ core.common_opts.ShowHelpOptions(),
+ core.common_opts.DumpDebugOptions(),
+ core.common_opts.AuthOptions(),
+ core.common_opts.YaBin3Options(),
+ ],
+ )
+
+ @staticmethod
+ def _choose_qt_handler(params):
+ if params.run:
+ ide.qt.run_qtcreator(params)
+ elif params.remote_host:
+ ide.remote_ide_qt.generate_remote_project(params)
+ else:
+ ide.qt.gen_qt_project(params)
diff --git a/devtools/ya/handlers/ide/ya.make b/devtools/ya/handlers/ide/ya.make
new file mode 100644
index 0000000000..c0802be535
--- /dev/null
+++ b/devtools/ya/handlers/ide/ya.make
@@ -0,0 +1,31 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.ide
+ __init__.py
+)
+
+PEERDIR(
+ devtools/ya/app
+ devtools/ya/build/build_opts
+ devtools/ya/core
+ devtools/ya/core/config
+ devtools/ya/core/yarg
+ devtools/ya/ide
+ devtools/ya/yalibrary/platform_matcher
+)
+
+IF (NOT YA_OPENSOURCE)
+ PEERDIR(
+ devtools/ya/ide/fsnotifier
+ )
+ENDIF()
+
+END()
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/java/__init__.py b/devtools/ya/handlers/java/__init__.py
new file mode 100644
index 0000000000..5c4c7c44e3
--- /dev/null
+++ b/devtools/ya/handlers/java/__init__.py
@@ -0,0 +1,49 @@
+from __future__ import absolute_import
+from . import helpers
+
+import app
+import core.yarg
+from build import build_opts
+import devtools.ya.test.opts as test_opts
+
+
+def default_options():
+ return [
+ build_opts.BuildTargetsOptions(with_free=True),
+ build_opts.BeVerboseOptions(),
+ build_opts.ShowHelpOptions(),
+ build_opts.YMakeDebugOptions(),
+ build_opts.YMakeBinOptions(),
+ build_opts.YMakeRetryOptions(),
+ build_opts.FlagsOptions(),
+ ]
+
+
+class JavaYaHandler(core.yarg.CompositeHandler):
+ def __init__(self):
+ super(JavaYaHandler, self).__init__(description='Java build helpers')
+
+ self['dependency-tree'] = core.yarg.OptsHandler(
+ action=app.execute(action=helpers.print_ymake_dep_tree),
+ description='Print dependency tree',
+ opts=default_options() + [build_opts.BuildTypeOptions('release')],
+ visible=True,
+ )
+ self['classpath'] = core.yarg.OptsHandler(
+ action=app.execute(action=helpers.print_classpath),
+ description='Print classpath',
+ opts=default_options() + [build_opts.BuildTypeOptions('release')],
+ visible=True,
+ )
+ self['test-classpath'] = core.yarg.OptsHandler(
+ action=app.execute(action=helpers.print_test_classpath),
+ description='Print run classpath for test module',
+ opts=default_options() + [test_opts.RunTestOptions()],
+ visible=True,
+ )
+ self['find-all-paths'] = core.yarg.OptsHandler(
+ action=app.execute(action=helpers.find_all_paths),
+ description='Find all PEERDIR paths of between two targets',
+ opts=default_options() + [build_opts.FindPathOptions()],
+ visible=True,
+ )
diff --git a/devtools/ya/handlers/java/helpers.py b/devtools/ya/handlers/java/helpers.py
new file mode 100644
index 0000000000..8a5a9cd35a
--- /dev/null
+++ b/devtools/ya/handlers/java/helpers.py
@@ -0,0 +1,250 @@
+from __future__ import absolute_import
+from __future__ import print_function
+import time
+import sys
+
+import core.yarg
+import build.build_opts as build_opts
+import build.graph as build_graph
+import build.ya_make as ya_make
+from build.build_facade import gen_managed_dep_tree, gen_targets_classpath
+from exts.tmp import temp_dir
+import yalibrary.formatter as yaformatter
+
+from jbuild.gen import base
+from jbuild.gen import consts
+from six.moves import map
+
+
+def fix_windows(path):
+ return path.replace('\\', '/')
+
+
+def get_java_ctx_with_tests(opts):
+ jopts = core.yarg.merge_opts(build_opts.ya_make_options(free_build_targets=True)).params()
+ jopts.dump_sources = True
+ jopts.create_symlinks = False
+ jopts.__dict__.update(opts.__dict__)
+ jopts.debug_options.append('x')
+ jopts.flags["YA_IDE_IDEA"] = "yes"
+
+ import app_ctx # XXX: via args
+
+ _, tests, _, ctx, _ = build_graph.build_graph_and_tests(
+ jopts, check=True, ev_listener=ya_make.get_print_listener(jopts, app_ctx.display), display=app_ctx.display
+ )
+
+ return ctx, tests
+
+
+def get_java_ctx(opts):
+ ctx, _ = get_java_ctx_with_tests(opts)
+ return ctx
+
+
+NORMAL = '[[imp]]{path}[[rst]]'
+EXCLUDED = '[[unimp]]{path} (omitted because of [[c:red]]EXCLUDE[[unimp]])[[rst]]'
+CONFLICT = '[[unimp]]{path} (omitted because of [[c:yellow]]conflict with {conflict_resolution}[[unimp]])[[rst]]'
+MANAGED = '[[imp]]{path}[[unimp]] (replaced from [[c:blue]]{orig}[[unimp]] because of [[c:blue]]DEPENDENCY_MANAGEMENT[[unimp]])[[rst]]'
+IGNORED = '[[unimp]]{path} (omitted as contrib proxy library)[[rst]]'
+DUPLICATE = '[[unimp]]{path} (*)[[rst]]'
+DIRECT_MANAGED = '[[imp]]{path}[[unimp]] (replaced from [[c:blue]]unversioned[[unimp]] because of [[c:blue]]DEPENDENCY_MANAGEMENT[[unimp]])[[rst]]'
+DIRECT_DEFAULT = (
+ '[[imp]]{path}[[unimp]] (replaced from [[c:magenta]]unversioned[[unimp]] to [[c:magenta]]default[[unimp]])[[rst]]'
+)
+
+
+def arrow_str(length):
+ s = ''
+
+ if length:
+ s += '| ' * (length - 1) + '|-->'
+
+ return '[[unimp]]' + s + '[[rst]]'
+
+
+def node_str(dep, excluded=False, expanded_above=False):
+ if expanded_above:
+ return DUPLICATE.format(path=dep.path)
+
+ elif excluded:
+ return EXCLUDED.format(path=dep.path)
+
+ elif dep.omitted_for_conflict:
+ return CONFLICT.format(path=dep.path, conflict_resolution=base.basename_unix_path(dep.conflict_resolution_path))
+
+ elif dep.replaced_for_dependency_management:
+ return MANAGED.format(path=dep.path, orig=base.basename_unix_path(dep.old_path_for_dependency_management))
+
+ elif dep.is_managed:
+ return DIRECT_MANAGED.format(path=dep.path)
+
+ elif dep.is_default:
+ return DIRECT_DEFAULT.format(path=dep.path)
+
+ else:
+ return NORMAL.format(path=dep.path)
+
+
+def graph_line(depth, dep, excluded=False, expanded_above=False):
+ return arrow_str(depth) + node_str(dep, excluded=excluded, expanded_above=expanded_above)
+
+
+def raise_not_a_java_path(path):
+ raise WrongInputException('{} is not a java module'.format(path))
+
+
+class WrongInputException(Exception):
+ mute = True
+
+
+def print_ymake_dep_tree(opts):
+ with temp_dir() as tmp:
+ res, evlog = gen_managed_dep_tree(
+ build_root=tmp,
+ build_type=opts.build_type,
+ build_targets=opts.abs_targets,
+ debug_options=opts.debug_options,
+ flags=opts.flags,
+ ymake_bin=opts.ymake_bin,
+ )
+
+ import app_ctx
+
+ ev_listener = ya_make.get_print_listener(opts, app_ctx.display)
+ for ev in evlog:
+ ev_listener(ev)
+
+ formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty())
+ print(formatter.format_message(res.stdout))
+ return any('Type' in ev and ev['Type'] == 'Error' for ev in evlog)
+
+
+def print_classpath(opts):
+ opts.flags['TRAVERSE_RECURSE'] = 'no'
+ opts.flags['TRAVERSE_RECURS_FOR_TEST'] = 'no'
+ with temp_dir() as tmp:
+ res, evlog = gen_targets_classpath(
+ build_root=tmp,
+ build_type=opts.build_type,
+ build_targets=opts.abs_targets,
+ debug_options=opts.debug_options,
+ flags=opts.flags,
+ ymake_bin=opts.ymake_bin,
+ )
+
+ import app_ctx
+
+ ev_listener = ya_make.get_print_listener(opts, app_ctx.display)
+ for ev in evlog:
+ ev_listener(ev)
+
+ formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty())
+ print(formatter.format_message(res.stdout))
+ return any('Type' in ev and ev['Type'] == 'Error' for ev in evlog)
+
+
+def print_test_classpath(opts):
+ opts.flags['IGNORE_JAVA_DEPENDENCIES_CONFIGURATION'] = 'yes'
+ opts.run_tests = 3
+ ctx, tests = get_java_ctx_with_tests(opts)
+ remaining_roots = set(map(fix_windows, opts.rel_targets))
+
+ formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty())
+ for test in tests:
+ if test.project_path not in remaining_roots:
+ continue
+ classpath = test.get_classpath()
+ if classpath is not None:
+ classpath = [item[len(consts.BUILD_ROOT) + 1 :] for item in classpath]
+ print(formatter.format_message("[[imp]]{}[[rst]]:\n\t{}".format(test.project_path, '\n\t'.join(classpath))))
+ remaining_roots.discard(test.project_path)
+
+ for path in remaining_roots:
+ print(formatter.format_message("[[imp]]{}[[rst]] is not a Java test".format(path)))
+
+
+def iter_all_routes(ctx, src_path, dest_path, route_callback, max_dist=None):
+ achievable = {}
+
+ def dest_achievable_from(path):
+ if path not in achievable:
+ achievable[path] = path == dest_path or any(dest_achievable_from(d.path) for d in ctx.by_path[path].deps)
+
+ return achievable[path]
+
+ stack = []
+
+ def visit(path):
+ stack.append(path)
+
+ if path == dest_path:
+ route_callback(stack)
+ stack.pop()
+ return # no loops
+
+ if max_dist is not None and len(stack) - 1 >= max_dist:
+ stack.pop()
+ return
+
+ for dep in ctx.by_path[path].deps:
+ dep_path = dep.path
+
+ if dest_achievable_from(dep_path):
+ visit(dep_path)
+
+ stack.pop()
+
+ if dest_achievable_from(src_path):
+ visit(src_path)
+
+
+def print_route(route):
+ dist = len(route) - 1
+
+ if dist < 4:
+ s = '[[c:green]]{}[[rst]]: '.format(dist)
+ elif dist < 7:
+ s = '[[c:yellow]]{}[[rst]]: '.format(dist)
+ else:
+ s = '[[c:red]]{}[[rst]]: '.format(dist)
+
+ s += '\n[[unimp]]-->'.join(['[[imp]]' + p for p in route])
+
+ return s
+
+
+def find_all_paths(opts):
+ start_time = time.time()
+ opts.flags['IGNORE_JAVA_DEPENDENCIES_CONFIGURATION'] = 'yes'
+ ctx = get_java_ctx(opts)
+
+ if len(opts.rel_targets) != 2:
+ raise WrongInputException(
+ 'Expected: <from> <to>\n' + 'Got: {}'.format(' '.join(['<' + p + '>' for p in opts.rel_targets]))
+ )
+
+ src_path = fix_windows(opts.rel_targets[0])
+ dest_path = fix_windows(opts.rel_targets[1])
+
+ if src_path not in ctx.by_path:
+ raise_not_a_java_path(src_path)
+
+ if dest_path not in ctx.by_path:
+ raise_not_a_java_path(dest_path)
+
+ count = [0]
+
+ formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty())
+
+ def route_callback(route):
+ print(formatter.format_message(print_route(route)))
+ count[0] += 1
+
+ try:
+ iter_all_routes(ctx, src_path, dest_path, route_callback, max_dist=opts.max_dist)
+ finally:
+ report = 'Found [[c:{}]]{}[[rst]] paths in [[c:yellow]]{}[[rst]] seconds'.format(
+ 'green' if count[0] else 'red', count[0], time.time() - start_time
+ )
+ print(formatter.format_message(report))
diff --git a/devtools/ya/handlers/java/ya.make b/devtools/ya/handlers/java/ya.make
new file mode 100644
index 0000000000..96140c1bf1
--- /dev/null
+++ b/devtools/ya/handlers/java/ya.make
@@ -0,0 +1,24 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.java
+ __init__.py
+ helpers.py
+)
+
+PEERDIR(
+ devtools/ya/app
+ devtools/ya/core/yarg
+ devtools/ya/build
+ devtools/ya/build/build_opts
+ devtools/ya/test/opts
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/make/__init__.py b/devtools/ya/handlers/make/__init__.py
new file mode 100644
index 0000000000..e9d774f562
--- /dev/null
+++ b/devtools/ya/handlers/make/__init__.py
@@ -0,0 +1,32 @@
+from __future__ import absolute_import
+from build.build_handler import do_ya_make
+from build.build_opts import ya_make_options
+
+import core.yarg
+
+import app
+
+
+class MakeYaHandler(core.yarg.OptsHandler):
+ description = 'Build and run tests\nTo see more help use [[imp]]-hh[[rst]]/[[imp]]-hhh[[rst]]'
+
+ def __init__(self):
+ core.yarg.OptsHandler.__init__(
+ self,
+ action=app.execute(action=do_ya_make),
+ examples=[
+ core.yarg.UsageExample('{prefix} -r', 'Build current directory in release mode', good_looking=100),
+ core.yarg.UsageExample(
+ '{prefix} -t -j16 library', 'Build and test library with 16 threads', good_looking=99
+ ),
+ core.yarg.UsageExample(
+ '{prefix} --checkout -j0', 'Checkout absent directories without build', good_looking=98
+ ),
+ ],
+ description=self.description,
+ opts=ya_make_options(
+ free_build_targets=True,
+ strip_idle_build_results=True,
+ ),
+ visible=True,
+ )
diff --git a/devtools/ya/handlers/make/ya.make b/devtools/ya/handlers/make/ya.make
new file mode 100644
index 0000000000..275750e1f7
--- /dev/null
+++ b/devtools/ya/handlers/make/ya.make
@@ -0,0 +1,26 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.make
+ __init__.py
+)
+
+PEERDIR(
+ devtools/ya/app
+ devtools/ya/build
+ devtools/ya/build/build_opts
+ devtools/ya/core/yarg
+)
+
+END()
+
+RECURSE(
+ tests
+)
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/maven_import/__init__.py b/devtools/ya/handlers/maven_import/__init__.py
new file mode 100644
index 0000000000..f1296c0c83
--- /dev/null
+++ b/devtools/ya/handlers/maven_import/__init__.py
@@ -0,0 +1,31 @@
+from __future__ import absolute_import
+import core.yarg
+import jbuild.maven.maven_import as mi
+import core.common_opts as common_opts
+import build.build_opts as build_opts
+import app
+import app_config
+
+from core.yarg.help_level import HelpLevel
+
+
+class MavenImportYaHandler(core.yarg.OptsHandler):
+ description = 'Import specified artifact from remote maven repository to arcadia/contrib/java.'
+ visible = app_config.in_house
+
+ def __init__(self):
+ super(MavenImportYaHandler, self).__init__(
+ action=app.execute(mi.do_import),
+ opts=[
+ build_opts.MavenImportOptions(visible=HelpLevel.BASIC),
+ build_opts.BuildThreadsOptions(build_threads=None),
+ common_opts.OutputStyleOptions(),
+ common_opts.TransportOptions(),
+ common_opts.ShowHelpOptions(),
+ build_opts.YMakeBinOptions(),
+ build_opts.CustomFetcherOptions(),
+ build_opts.ToolsOptions(),
+ ]
+ + build_opts.checkout_options(),
+ description=self.description,
+ )
diff --git a/devtools/ya/handlers/maven_import/ya.make b/devtools/ya/handlers/maven_import/ya.make
new file mode 100644
index 0000000000..c2e16c6d35
--- /dev/null
+++ b/devtools/ya/handlers/maven_import/ya.make
@@ -0,0 +1,28 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.maven_import
+ __init__.py
+)
+
+PEERDIR(
+ devtools/ya/app
+ devtools/ya/build/build_opts
+ devtools/ya/core
+ devtools/ya/core/yarg
+ devtools/ya/jbuild
+ devtools/ya/yalibrary/tools
+)
+
+END()
+
+RECURSE(
+ tests
+)
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/package/__init__.py b/devtools/ya/handlers/package/__init__.py
new file mode 100644
index 0000000000..adaf2980f1
--- /dev/null
+++ b/devtools/ya/handlers/package/__init__.py
@@ -0,0 +1,88 @@
+from __future__ import absolute_import
+import logging
+
+import app
+import core.yarg
+import core.common_opts
+import build.ya_make
+import build.targets_deref
+import build.build_opts
+import devtools.ya.test.opts as test_opts
+
+import package.docker
+import package.packager
+import devtools.ya.handlers.package.opts as package_opts
+from core.yarg.help_level import HelpLevel
+
+logger = logging.getLogger(__name__)
+
+
+class PackageYaHandler(core.yarg.OptsHandler):
+ description = """Build package using json package description in the release build type by default.
+For more info see https://docs.yandex-team.ru/ya-make/usage/ya_package"""
+
+ def __init__(self):
+ super(PackageYaHandler, self).__init__(
+ action=app.execute(package.packager.do_package, respawn=app.RespawnType.OPTIONAL),
+ description=self.description,
+ examples=[
+ core.yarg.UsageExample(
+ cmd='{prefix} <path to json description>',
+ description='Create tarball package from json description',
+ )
+ ],
+ opts=[
+ package_opts.PackageOperationalOptions(),
+ package_opts.PackageCustomizableOptions(),
+ package_opts.InterimOptions(),
+ core.common_opts.LogFileOptions(),
+ core.common_opts.EventLogFileOptions(),
+ build.build_opts.BuildTypeOptions('release'),
+ build.build_opts.BuildThreadsOptions(build_threads=None),
+ core.common_opts.CrossCompilationOptions(),
+ build.build_opts.ArcPrefetchOptions(),
+ build.build_opts.ContentUidsOptions(),
+ build.build_opts.KeepTempsOptions(),
+ build.build_opts.RebuildOptions(),
+ build.build_opts.StrictInputsOptions(),
+ build.build_opts.DumpReportOptions(),
+ build.build_opts.OutputOptions(),
+ build.build_opts.AuthOptions(),
+ build.build_opts.YMakeDumpGraphOptions(),
+ build.build_opts.YMakeDebugOptions(),
+ build.build_opts.YMakeBinOptions(),
+ build.build_opts.YMakeRetryOptions(),
+ build.build_opts.ExecutorOptions(),
+ build.build_opts.ForceDependsOptions(),
+ build.build_opts.IgnoreRecursesOptions(),
+ core.common_opts.CustomSourceRootOptions(),
+ core.common_opts.CustomBuildRootOptions(),
+ core.common_opts.ShowHelpOptions(),
+ core.common_opts.BeVerboseOptions(),
+ core.common_opts.HtmlDisplayOptions(),
+ core.common_opts.CommonUploadOptions(),
+ build.build_opts.SandboxUploadOptions(ssh_key_option_name="--ssh-key", visible=HelpLevel.BASIC),
+ build.build_opts.MDSUploadOptions(visible=HelpLevel.BASIC),
+ core.common_opts.TransportOptions(),
+ build.build_opts.CustomFetcherOptions(),
+ build.build_opts.DistCacheOptions(),
+ build.build_opts.FlagsOptions(),
+ build.build_opts.PGOOptions(),
+ test_opts.RunTestOptions(),
+ test_opts.DebuggingOptions(),
+ # strip_idle_build_results must be False to avoid removal of build nodes which are
+ # reachable due RECURSE and used in package, but not required for tests
+ test_opts.DepsOptions(strip_idle_build_results=False),
+ test_opts.FileReportsOptions(),
+ test_opts.FilteringOptions(test_size_filters=None),
+ test_opts.PytestOptions(),
+ test_opts.JUnitOptions(),
+ test_opts.RuntimeEnvironOptions(),
+ test_opts.TestToolOptions(),
+ test_opts.UidCalculationOptions(cache_tests=False),
+ core.common_opts.YaBin3Options(),
+ ]
+ + build.build_opts.distbs_options()
+ + build.build_opts.checkout_options()
+ + build.build_opts.svn_checkout_options(),
+ )
diff --git a/devtools/ya/handlers/package/opts/__init__.py b/devtools/ya/handlers/package/opts/__init__.py
new file mode 100644
index 0000000000..e65532ef6f
--- /dev/null
+++ b/devtools/ya/handlers/package/opts/__init__.py
@@ -0,0 +1,691 @@
+import collections
+import logging
+
+import build.build_opts
+import core.yarg
+import devtools.ya.test.opts as test_opts
+from devtools.ya.package import const
+
+logger = logging.getLogger(__name__)
+
+
+COMMON_SUBGROUP = core.yarg.Group('Common', 1)
+TAR_SUBGROUP = core.yarg.Group('Tar', 2)
+DEB_SUBGROUP = core.yarg.Group('Debian', 3)
+DOCKER_SUBGROUP = core.yarg.Group('Docker', 4)
+AAR_SUBGROUP = core.yarg.Group('Aar', 5)
+RPM_SUBGROUP = core.yarg.Group('Rpm', 6)
+NPM_SUBGROUP = core.yarg.Group('Npm', 7)
+PYTHON_WHEEL_SUBGROUP = core.yarg.Group('Python wheel', 8)
+
+
+class PackageOperationalOptions(core.yarg.Options):
+ def __init__(self):
+ self.artifactory_password_path = None
+ self.build_debian_scripts = False
+ self.build_only = False
+ self.change_log = None
+ self.cleanup = True
+ self.codec = None
+ self.convert = None
+ self.custom_data_root = None
+ self.custom_tests_data_root = None
+ self.debian_distribution = 'unstable'
+ self.debian_upload_token = None # please, do not remove, we really need it in opensource nebius ya
+ self.docker_no_cache = False
+ self.docker_push_image = False
+ self.docker_remote_image_version = None
+ self.docker_use_remote_cache = False
+ self.dump_build_targets = None
+ self.dump_inputs = None
+ self.ignore_fail_tests = False
+ self.list_codecs = False
+ self.nanny_release = None
+ self.package_output = None
+ self.packages = []
+ self.publish_to = {}
+ self.raw_package_path = None
+ self.run_long_tests = False
+ self.sandbox_download_protocols = []
+ self.upload = False
+ self.wheel_access_key_path = None
+ self.wheel_secret_key_path = None
+
+ @staticmethod
+ def consumer():
+ return [
+ core.yarg.ArgConsumer(
+ names=['--publish-to'],
+ help='Publish package to the specified dist',
+ hook=core.yarg.DictPutHook('publish_to'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--build-only'],
+ hook=core.yarg.SetConstValueHook('build_only', True),
+ visible=False,
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--change-log'],
+ help='Change log text or path to the existing changelog file',
+ hook=core.yarg.SetValueHook('change_log'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--new'],
+ help='Use new ya package json format',
+ hook=core.yarg.SetConstValueHook('convert', False),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--old'],
+ help='Use old ya package json format',
+ hook=core.yarg.SetConstValueHook('convert', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--tests-data-root'],
+ help="Custom location for arcadia_tests_data dir, defaults to <source root>/../arcadia_tests_data",
+ hook=core.yarg.SetValueHook('custom_tests_data_root'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--data-root'],
+ help="Custom location for data dir, defaults to <source root>/../data",
+ hook=core.yarg.SetValueHook('custom_data_root'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--artifactory-password-path'],
+ help='Path to file with artifactory password',
+ hook=core.yarg.SetValueHook('artifactory_password_path'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.EnvConsumer(
+ 'YA_ARTIFACTORY_PASSWORD_PATH',
+ help='Path to file with artifactory password',
+ hook=core.yarg.SetValueHook('artifactory_password_path'),
+ ),
+ core.yarg.ArgConsumer(
+ names=['--dump-arcadia-inputs'],
+ help='Only dump inputs, do not build package',
+ hook=core.yarg.SetValueHook('dump_inputs'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--ignore-fail-tests'],
+ help='Create package, no matter tests failed or not',
+ hook=core.yarg.SetConstValueHook('ignore_fail_tests', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--codec'],
+ help='Codec name for uc compression',
+ hook=core.yarg.SetValueHook('codec'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--codecs-list'],
+ help='Show available codecs for --uc',
+ hook=core.yarg.SetConstValueHook('list_codecs', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ["-O", "--package-output"],
+ help="Specifies directory for package output",
+ hook=core.yarg.SetValueHook('package_output'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.FreeArgConsumer(
+ help='Package description file name(s)',
+ hook=core.yarg.SetValueHook(name='packages'),
+ ),
+ core.yarg.ArgConsumer(
+ ['--sandbox-download-protocol'],
+ help='Sandbox download protocols comma-separated (default: http,http_tgz)',
+ hook=core.yarg.SetValueHook(
+ 'sandbox_download_protocols', transform=lambda val: [_f for _f in val.split(",") if _f]
+ ),
+ visible=False,
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--wheel-repo-access-key'],
+ help='Path to access key for wheel repository',
+ hook=core.yarg.SetValueHook('wheel_access_key_path'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=PYTHON_WHEEL_SUBGROUP,
+ ),
+ core.yarg.EnvConsumer(
+ 'YA_WHEEL_REPO_ACCESS_KEY_PATH',
+ help='Path to access key for wheel repository',
+ hook=core.yarg.SetValueHook('wheel_access_key_path'),
+ ),
+ core.yarg.ArgConsumer(
+ names=['--wheel-repo-secret-key'],
+ help='Path to secret key for wheel repository',
+ hook=core.yarg.SetValueHook('wheel_secret_key_path'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=PYTHON_WHEEL_SUBGROUP,
+ ),
+ core.yarg.EnvConsumer(
+ 'YA_WHEEL_SECRET_KEY_PATH',
+ help='Path to secret key for wheel repository',
+ hook=core.yarg.SetValueHook('wheel_secret_key_path'),
+ ),
+ core.yarg.ArgConsumer(
+ names=['--raw-package-path'],
+ help="Custom path for raw-package (implies --raw-package)",
+ hook=core.yarg.SetValueHook('raw_package_path'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--no-cleanup'],
+ help='Do not clean the temporary directory',
+ hook=core.yarg.SetConstValueHook('cleanup', False),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--build-debian-scripts'],
+ hook=core.yarg.SetConstValueHook('build_debian_scripts', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--debian-distribution'],
+ help='Debian distribution',
+ hook=core.yarg.SetValueHook('debian_distribution'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.EnvConsumer(
+ 'YA_DEBIAN_UPLOAD_TOKEN',
+ help='Iam token or path to iam token for nebiuscloud debian repository',
+ hook=core.yarg.SetValueHook('debian_upload_token'),
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-push'],
+ help='Push docker image to registry',
+ hook=core.yarg.SetConstValueHook('docker_push_image', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-no-cache'],
+ help='Disable docker cache',
+ hook=core.yarg.SetConstValueHook('docker_no_cache', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ConfigConsumer("docker_no_cache"),
+ core.yarg.ArgConsumer(
+ names=['--dump-build-targets'],
+ hook=core.yarg.SetValueHook('dump_build_targets'),
+ visible=False,
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-use-remote-cache'],
+ help='Use image from registry as cache source',
+ hook=core.yarg.SetConstValueHook('docker_use_remote_cache', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-remote-image-version'],
+ help='Specify image version to be used as cache source',
+ hook=core.yarg.SetValueHook('docker_remote_image_version'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--nanny-release'],
+ help='Notify nanny about new release',
+ hook=core.yarg.SetValueHook(
+ 'nanny_release', transform=lambda s: s.upper(), values=const.NANNY_RELEASE_TYPES
+ ),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--upload'],
+ help='Upload created package to sandbox',
+ hook=core.yarg.SetConstValueHook('upload', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ ]
+
+ def postprocess(self):
+ if self.convert is not None:
+ logger.warning("Package format will be detected automatically, no need to use --new and --old")
+ if self.nanny_release and not self.docker_push_image:
+ raise core.yarg.ArgsValidatingException("Using --nanny-release without --docker-push is pointless")
+
+ def postprocess2(self, params):
+ if params.raw_package_path and not params.raw_package:
+ params.raw_package = True
+ # old params compatibility
+ if getattr(params, 'run_long_tests', False):
+ params.run_tests = test_opts.RunTestOptions.RunAllTests
+
+
+class PackageCustomizableOptions(core.yarg.Options):
+ """
+ Don't add parameters here by default, otherwise user could use them in package.json.
+ For more info see https://docs.yandex-team.ru/ya-make/usage/ya_package/json#params
+ """
+
+ deb_compression_levels = collections.OrderedDict(
+ sorted(
+ {
+ 'none': 0,
+ 'low': 3,
+ 'medium': 6,
+ 'high': 9,
+ }.items(),
+ key=lambda i: i[1],
+ )
+ )
+
+ def __init__(self):
+ self.arch_all = False
+ self.artifactory = None
+ self.compress_archive = True
+ self.compression_filter = None
+ self.compression_level = None
+ self.create_dbg = False
+ self.custom_version = None
+ self.debian_arch = None
+ self.debian_compression_level = None
+ self.debian_compression_type = 'gzip'
+ self.docker_add_host = []
+ self.docker_build_arg = {}
+ self.docker_build_network = None
+ self.docker_platform = None
+ self.docker_registry = "registry.yandex.net"
+ self.docker_repository = ""
+ self.docker_save_image = False
+ self.docker_secrets = []
+ self.docker_target = None
+ self.dupload_max_attempts = 1
+ self.dupload_no_mail = False
+ self.ensure_package_published = False
+ self.force_dupload = False
+ self.format = None
+ self.full_strip = False
+ self.key = None
+ self.overwrite_read_only_files = False
+ self.raw_package = False
+ self.resource_attrs = {}
+ self.resource_type = "YA_PACKAGE"
+ self.sandbox_task_id = 0
+ self.sign = True
+ self.sloppy_deb = False
+ self.store_debian = True
+ self.strip = False
+ self.wheel_platform = ""
+ self.wheel_python3 = False
+
+ @staticmethod
+ def consumer():
+ return [
+ core.yarg.ArgConsumer(
+ names=['--strip'],
+ help='Strip binaries (only debug symbols: "strip -g")',
+ hook=core.yarg.SetConstValueHook('strip', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--full-strip'],
+ help='Strip binaries',
+ hook=core.yarg.SetConstValueHook('full_strip', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--set-sandbox-task-id'],
+ visible=False,
+ help='Use the provided task id for the package version if needed',
+ hook=core.yarg.SetValueHook('sandbox_task_id', int),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--wheel-platform'],
+ visible=True,
+ help='Set wheel package platform',
+ hook=core.yarg.SetValueHook('wheel_platform'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=PYTHON_WHEEL_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--key'],
+ help='The key to use for signing',
+ hook=core.yarg.SetValueHook('key'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--debian'],
+ help='Build debian package',
+ hook=core.yarg.SetConstValueHook('format', const.PackageFormat.DEBIAN),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--tar'],
+ help='Build tarball package',
+ hook=core.yarg.SetConstValueHook('format', const.PackageFormat.TAR),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--no-compression'],
+ help="Don't compress tar archive (for --tar only)",
+ hook=core.yarg.SetConstValueHook('compress_archive', False),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--create-dbg'],
+ help='Create separate package with debug info (works only in case of --strip or --full-strip)',
+ hook=core.yarg.SetConstValueHook('create_dbg', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ["--compression-filter"],
+ help="Specifies compression filter (gzip/zstd)",
+ hook=core.yarg.SetValueHook('compression_filter'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ["--compression-level"],
+ help="Specifies compression level (0-9 for gzip [6 is default], 0-22 for zstd [3 is default])",
+ hook=core.yarg.SetValueHook('compression_level', transform=lambda s: int(s)),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker'],
+ help='Build docker',
+ hook=core.yarg.SetConstValueHook('format', const.PackageFormat.DOCKER),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--rpm'],
+ help='Build rpm package',
+ hook=core.yarg.SetConstValueHook('format', const.PackageFormat.RPM),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=RPM_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--aar'],
+ help='Build aar package',
+ hook=core.yarg.SetConstValueHook('format', const.PackageFormat.AAR),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=AAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--npm'],
+ help='Build npm package',
+ hook=core.yarg.SetConstValueHook('format', const.PackageFormat.NPM),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=NPM_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--wheel'],
+ help='Build wheel package',
+ hook=core.yarg.SetConstValueHook('format', const.PackageFormat.WHEEL),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=PYTHON_WHEEL_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--wheel-python3'],
+ help='use python3 when building wheel package',
+ hook=core.yarg.SetConstValueHook('wheel_python3', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=PYTHON_WHEEL_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--artifactory'],
+ help='Build package and upload it to artifactory',
+ hook=core.yarg.SetConstValueHook("artifactory", True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-add-host'],
+ help='Docker --add-host',
+ hook=core.yarg.SetAppendHook('docker_add_host'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-secret'],
+ help='Same as Docker --secret. You can pass few secrets at the same time',
+ hook=core.yarg.SetAppendHook('docker_secrets'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-registry'],
+ help='Docker registry',
+ hook=core.yarg.SetValueHook('docker_registry'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-repository'],
+ help='Specify private repository',
+ hook=core.yarg.SetValueHook('docker_repository'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-save-image'],
+ help='Save docker image to archive',
+ hook=core.yarg.SetConstValueHook('docker_save_image', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-network'],
+ help='--network parameter for `docker build` command',
+ hook=core.yarg.SetValueHook('docker_build_network'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-platform'],
+ help='Specify platform for docker build (require buildx)',
+ hook=core.yarg.SetValueHook('docker_platform'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-build-arg'],
+ help='--build-arg parameter for `docker build` command, set it in the <key>=<value> form',
+ hook=core.yarg.DictPutHook('docker_build_arg'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--docker-target'],
+ help='Specifying target build stage (--target)',
+ hook=core.yarg.SetValueHook('docker_target'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DOCKER_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--raw-package'],
+ help="Used with --tar to get package content without tarring",
+ hook=core.yarg.SetConstValueHook('raw_package', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=TAR_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--sloppy-and-fast-debian'],
+ help="Fewer checks and no compression when building debian package",
+ hook=core.yarg.SetConstValueHook('sloppy_deb', True),
+ visible=False,
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--not-sign-debian'],
+ help='Do not sign debian package',
+ hook=core.yarg.SetConstValueHook('sign', False),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--custom-version'],
+ help='Custom package version',
+ hook=core.yarg.SetValueHook('custom_version'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--debian-arch'],
+ help='Debian arch (passed to debuild as `-a`)',
+ hook=core.yarg.SetValueHook('debian_arch'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--arch-all'],
+ help='Use "Architecture: all" in debian',
+ hook=core.yarg.SetConstValueHook('arch_all', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--force-dupload'],
+ help='dupload --force',
+ hook=core.yarg.SetConstValueHook('force_dupload', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['-z', '--debian-compression'],
+ help="deb-file compresson level ({})".format(
+ ", ".join(list(PackageCustomizableOptions.deb_compression_levels.keys()))
+ ),
+ hook=core.yarg.SetValueHook(
+ 'debian_compression_level', values=list(PackageCustomizableOptions.deb_compression_levels.keys())
+ ),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['-Z', '--debian-compression-type'],
+ help="deb-file compression type used when building deb-file (allowed types: {}, gzip (default), xz, bzip2, lzma, none)".format(
+ const.DEBIAN_HOST_DEFAULT_COMPRESSION_LEVEL
+ ),
+ hook=core.yarg.SetValueHook('debian_compression_type'),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--dont-store-debian'],
+ help="Save debian package in a separate archive",
+ hook=core.yarg.SetConstValueHook('store_debian', False),
+ visible=False,
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--upload-resource-type'],
+ help='Created resource type',
+ hook=core.yarg.SetValueHook('resource_type'),
+ group=build.build_opts.SANDBOX_UPLOAD_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ ['--upload-resource-attr'],
+ help='Resource attr, set it in the <name>=<value> form',
+ hook=core.yarg.DictPutHook(name='resource_attrs'),
+ group=build.build_opts.SANDBOX_UPLOAD_OPT_GROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--dupload-max-attempts'],
+ help='How many times try to run dupload if it fails',
+ hook=core.yarg.SetValueHook('dupload_max_attempts', int),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--dupload-no-mail'],
+ help='dupload --no-mail',
+ hook=core.yarg.SetConstValueHook('dupload_no_mail', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=DEB_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--overwrite-read-only-files'],
+ help='Overwrite read-only files in package',
+ hook=core.yarg.SetConstValueHook('overwrite_read_only_files', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ core.yarg.ArgConsumer(
+ names=['--ensure-package-published'],
+ help='Ensure that package is available in the repository',
+ hook=core.yarg.SetConstValueHook('ensure_package_published', True),
+ group=core.yarg.PACKAGE_OPT_GROUP,
+ subgroup=COMMON_SUBGROUP,
+ ),
+ ]
+
+ def postprocess(self):
+ if self.debian_compression_level is not None:
+ self.debian_compression_level = self.deb_compression_levels[self.debian_compression_level]
+ if self.create_dbg:
+ if not self.full_strip:
+ self.strip = True
+ if self.compression_filter not in (None, 'gzip', 'zstd'):
+ raise core.yarg.ArgsValidatingException(
+ "Using unsupported compression filter: {}".format(self.compression_filter)
+ )
+
+
+class InterimOptions(core.yarg.Options):
+ Visible = False
+
+ def __init__(self):
+ self.verify_patterns_usage = True
+
+ # All this options
+ # - !! should never be available in YA_PACKAGE sandbox task !!
+ # - will be removed when work is done
+ def consumer(self):
+ return [
+ core.yarg.ArgConsumer(
+ names=['--fixme-CHEMODAN-80080'],
+ help='See CHEMODAN-80080 and DEVTOOLSSUPPORT-12411 for more info',
+ hook=core.yarg.SetConstValueHook('verify_patterns_usage', False),
+ visible=self.Visible,
+ ),
+ ]
diff --git a/devtools/ya/handlers/package/opts/ya.make b/devtools/ya/handlers/package/opts/ya.make
new file mode 100644
index 0000000000..f13c20f4b5
--- /dev/null
+++ b/devtools/ya/handlers/package/opts/ya.make
@@ -0,0 +1,19 @@
+PY23_LIBRARY()
+
+PY_SRCS(
+ __init__.py
+)
+
+PEERDIR(
+ devtools/ya/build/build_opts
+ devtools/ya/core
+ devtools/ya/core/yarg
+ devtools/ya/exts
+ devtools/ya/package/const
+ devtools/ya/test/const
+ devtools/ya/test/opts
+)
+
+STYLE_PYTHON()
+
+END()
diff --git a/devtools/ya/handlers/package/ya.make b/devtools/ya/handlers/package/ya.make
new file mode 100644
index 0000000000..98c8a47dd9
--- /dev/null
+++ b/devtools/ya/handlers/package/ya.make
@@ -0,0 +1,27 @@
+PY23_LIBRARY()
+
+PY_SRCS(
+ NAMESPACE handlers.package
+ __init__.py
+)
+
+PEERDIR(
+ contrib/python/pathlib2
+ devtools/ya/app
+ devtools/ya/build
+ devtools/ya/build/build_opts
+ devtools/ya/core
+ devtools/ya/core/yarg
+ devtools/ya/handlers/package/opts
+ devtools/ya/package
+ devtools/ya/test/opts
+)
+
+STYLE_PYTHON()
+
+END()
+
+RECURSE_FOR_TESTS(
+ bin
+ opts
+)
diff --git a/devtools/ya/handlers/test/__init__.py b/devtools/ya/handlers/test/__init__.py
new file mode 100644
index 0000000000..2b3b2e4de4
--- /dev/null
+++ b/devtools/ya/handlers/test/__init__.py
@@ -0,0 +1,40 @@
+import app
+
+from build.build_handler import do_ya_make
+from build.build_opts import ya_make_options
+
+import core.yarg
+
+
+class YaTestYaHandler(core.yarg.OptsHandler):
+ description = 'Build and run all tests\n[[imp]]ya test[[rst]] is alias for [[imp]]ya make -A[[rst]]'
+
+ def __init__(self):
+ core.yarg.OptsHandler.__init__(
+ self,
+ action=app.execute(action=do_ya_make),
+ examples=[
+ core.yarg.UsageExample(
+ '{prefix}',
+ 'Build and run all tests',
+ good_looking=101,
+ ),
+ core.yarg.UsageExample(
+ '{prefix} -t',
+ 'Build and run small tests only',
+ good_looking=102,
+ ),
+ core.yarg.UsageExample(
+ '{prefix} -tt',
+ 'Build and run small and medium tests',
+ good_looking=103,
+ ),
+ ],
+ description=self.description,
+ opts=ya_make_options(
+ free_build_targets=True,
+ is_ya_test=True,
+ strip_idle_build_results=True,
+ ),
+ visible=True,
+ )
diff --git a/devtools/ya/handlers/test/ya.make b/devtools/ya/handlers/test/ya.make
new file mode 100644
index 0000000000..70e9dee8a1
--- /dev/null
+++ b/devtools/ya/handlers/test/ya.make
@@ -0,0 +1,21 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.test
+ __init__.py
+)
+
+PEERDIR(
+ devtools/ya/app
+ devtools/ya/build
+ devtools/ya/build/build_opts
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/tool/__init__.py b/devtools/ya/handlers/tool/__init__.py
new file mode 100644
index 0000000000..b4ac4ccdca
--- /dev/null
+++ b/devtools/ya/handlers/tool/__init__.py
@@ -0,0 +1,281 @@
+from __future__ import absolute_import
+from __future__ import print_function
+import os
+import sys
+import logging
+
+from core.common_opts import CrossCompilationOptions
+from core.yarg import (
+ ArgConsumer,
+ CompositeHandler,
+ EnvConsumer,
+ SetConstValueHook,
+ SetValueHook,
+ Options,
+ OptsHandler,
+ FreeArgConsumer,
+ ConfigConsumer,
+ ExtendHook,
+ ShowHelpException,
+ SetAppendHook,
+ BaseHook,
+)
+
+import app
+
+from build.build_opts import CustomFetcherOptions, SandboxAuthOptions, ToolsOptions
+from core.yarg.groups import PRINT_CONTROL_GROUP
+from core.yarg.help_level import HelpLevel
+from yalibrary.tools import environ, param, resource_id, task_id, tool, tools, toolchain_root, toolchain_sys_libs
+from yalibrary.toolscache import lock_resource
+from yalibrary.platform_matcher import is_darwin_arm64
+import core.config
+import core.respawn
+import exts.process
+import exts.windows
+
+logger = logging.getLogger(__name__)
+
+
+class ToolYaHandler(CompositeHandler):
+ description = 'Execute specific tool'
+
+ @staticmethod
+ def common_download_options():
+ return [SandboxAuthOptions(), CustomFetcherOptions(), ToolsOptions()]
+
+ def __init__(self):
+ CompositeHandler.__init__(
+ self,
+ description=self.description,
+ examples=[
+ core.yarg.UsageExample('{prefix} --ya-help', 'Print yatool specific options', good_looking=20),
+ core.yarg.UsageExample('{prefix} --print-path', 'Print path to tool executable file', good_looking=10),
+ core.yarg.UsageExample(
+ '{prefix} --force-update',
+ 'Check tool for updates before the update interval elapses',
+ good_looking=10,
+ ),
+ ],
+ )
+ for x in tools():
+ self[x.name] = OptsHandler(
+ action=app.execute(action=do_tool, respawn=app.RespawnType.OPTIONAL),
+ description=x.description,
+ visible=x.visible,
+ opts=[ToolOptions(x.name)] + self.common_download_options(),
+ unknown_args_as_free=True,
+ )
+
+
+class DummyHook(BaseHook):
+ def __call__(self, to, *args):
+ # type: ("Options", tp.Optional[tp.Any]) -> None
+ pass
+
+ @staticmethod
+ def need_value():
+ return False
+
+
+class ToolOptions(Options):
+ def __init__(self, tool):
+ Options.__init__(self)
+ self.tool = tool
+ self.print_path = None
+ self.print_toolchain_path = None
+ self.print_toolchain_sys_libs = None
+ self.toolchain = None
+ self.param = None
+ self.platform = None
+ self.target_platforms = []
+ self.need_task_id = None
+ self.need_resource_id = None
+ self.show_help = False
+ self.tail_args = []
+ self.host_platform = None
+ self.hide_arm64_host_warning = False
+ self.force_update = False
+ self.force_refetch = False
+
+ @staticmethod
+ def consumer():
+ return [
+ ArgConsumer(
+ ['--print-path'],
+ help='Only print path to tool, do not execute',
+ hook=SetConstValueHook('print_path', True),
+ ),
+ ArgConsumer(
+ ['--print-toolchain-path'],
+ help='Print path to toolchain root',
+ hook=SetConstValueHook('print_toolchain_path', True),
+ ),
+ ArgConsumer(
+ ['--print-toolchain-sys-libs'],
+ help='Print pathes to toolchsin system libraries',
+ hook=SetConstValueHook('print_toolchain_sys_libs', True),
+ ),
+ ArgConsumer(['--platform'], help="Set specific platform", hook=SetValueHook('platform')),
+ ArgConsumer(['--host-platform'], help="Set host platform", hook=SetValueHook('host_platform')),
+ EnvConsumer('YA_TOOL_HOST_PLATFORM', hook=SetValueHook('host_platform')),
+ ArgConsumer(['--toolchain'], help="Specify toolchain", hook=SetValueHook('toolchain')),
+ ArgConsumer(['--get-param'], help="Get specified param", hook=SetValueHook('param')),
+ ArgConsumer(
+ ['--get-resource-id'],
+ help="Get resource id for specific platform (the platform should be specified)",
+ hook=SetConstValueHook('need_resource_id', True),
+ ),
+ ArgConsumer(['--get-task-id'], help="Get task id", hook=SetConstValueHook('need_task_id', True)),
+ ArgConsumer(['--ya-help'], help="Show help", hook=SetConstValueHook('show_help', True)),
+ ArgConsumer(
+ ['--target-platform'],
+ help='Target platform',
+ hook=SetAppendHook('target_platforms', values=CrossCompilationOptions.generate_target_platforms_cxx()),
+ ),
+ ArgConsumer(
+ ['--hide-arm64-host-warning'],
+ help='Hide MacOS arm64 host warning',
+ hook=SetConstValueHook('hide_arm64_host_warning', True),
+ group=PRINT_CONTROL_GROUP,
+ visible=HelpLevel.EXPERT if is_darwin_arm64() else False,
+ ),
+ EnvConsumer('YA_TOOL_HIDE_ARM64_HOST_WARNING', hook=SetConstValueHook('hide_arm64_host_warning', True)),
+ ConfigConsumer('hide_arm64_host_warning'),
+ ArgConsumer(
+ ['--force-update'],
+ help='Check tool for updates before the update interval elapses',
+ hook=SetConstValueHook('force_update', True),
+ ),
+ ArgConsumer(['--force-refetch'], help='Refetch toolchain', hook=SetConstValueHook('force_refetch', True)),
+ ArgConsumer(['--print-fastpath-error'], help='Print fast path failure error', hook=DummyHook()),
+ FreeArgConsumer(help='arg', hook=ExtendHook(name='tail_args')),
+ ]
+
+ def postprocess(self):
+ if self.show_help:
+ raise ShowHelpException()
+ if self.toolchain and self.target_platforms:
+ raise core.yarg.ArgsValidatingException("Do not use --toolchain and --target-platform args together")
+ if self.force_update:
+ os.environ['YA_TOOL_FORCE_UPDATE'] = "1"
+
+
+def _replace(s, transformations):
+ for k, v in transformations.items():
+ s = s.replace('$({})'.format(k), v)
+ return s
+
+
+def _useful_env_vars():
+ return {'YA_TOOL': sys.argv[0]}
+
+
+def do_tool(params):
+ tool_name = params.tool
+ extra_args = params.tail_args
+ target_platform = params.target_platforms
+ host_platform = params.host_platform
+ if target_platform:
+ if len(target_platform) > 1:
+ raise Exception('Multiple target platforms are not supported by this code for now')
+ target_platform = target_platform[0]
+ else:
+ target_platform = None
+
+ if is_darwin_arm64() and not host_platform:
+ host_platform = 'darwin'
+ if not params.hide_arm64_host_warning:
+ try:
+ import app_ctx
+
+ app_ctx.display.emit_message("You use x86_64 version of selected tool.")
+ except Exception as e:
+ logger.error("Can't print arm64 warning message: {}".format(e))
+
+ tool_path = tool(
+ tool_name,
+ params.toolchain,
+ target_platform=target_platform,
+ for_platform=host_platform,
+ force_refetch=params.force_refetch,
+ )
+ if exts.windows.on_win() and not tool_path.endswith('.exe'): # XXX: hack. Think about ya.conf.json format
+ logger.debug('Rename tool for win: %s', tool_path)
+ tool_path += '.exe'
+
+ lock_result = False
+ for_platform = params.platform or params.host_platform or None
+
+ if params.need_task_id:
+ tid = task_id(tool_name, params.toolchain)
+ if tid is not None:
+ print(tid)
+ else:
+ raise Exception("Tool '{}' has no task id".format(tool_name))
+ elif params.need_resource_id:
+ print(resource_id(tool_name, params.toolchain, for_platform))
+ elif params.param:
+ print(param(tool_name, params.toolchain, params.param))
+ elif params.print_toolchain_path:
+ print(toolchain_root(tool_name, params.toolchain, for_platform))
+ lock_result = True
+ elif params.print_toolchain_sys_libs:
+ print(toolchain_sys_libs(tool_name, params.toolchain, for_platform))
+ lock_result = True
+ elif params.print_path:
+ print(tool_path)
+ lock_result = True
+ elif os.path.isfile(tool_path):
+ env = core.respawn.filter_env(os.environ.copy())
+
+ # Remove environment variables set by 'ya' wrapper.
+ # They are actually one-time ya-bin parameters rather than inheritable environment
+ # for all descendant processes.
+ for key in ('YA_SOURCE_ROOT', 'YA_PYVER_REQUIRE', 'YA_PYVER_SET_FORCED'):
+ env.pop(key, None)
+
+ env.update(_useful_env_vars())
+ for key, value in environ(tool_name, params.toolchain).items():
+ env[key] = _replace(
+ os.pathsep.join(value), {'ROOT': toolchain_root(tool_name, params.toolchain, for_platform)}
+ )
+ if tool_name == 'gdb':
+ # gdb does not fit in 8 MB stack with large cores (DEVTOOLS-5040).
+ try:
+ import resource as r
+ except ImportError:
+ pass
+ else:
+ soft, hard = r.getrlimit(r.RLIMIT_STACK)
+ new = 128 << 20
+ logger.debug("Limit info: soft=%d hard=%d new=%d", soft, hard, new)
+ if hard != r.RLIM_INFINITY:
+ new = min(new, hard)
+ logger.debug("Limit info: new=%d", new)
+ if new > soft:
+ logger.debug("Limit info: setting new limits=(%d, %d)", new, hard)
+ try:
+ r.setrlimit(r.RLIMIT_STACK, (new, hard))
+ except ValueError as e:
+ logger.error("Failure while setting RLIMIT_STACK ({}, {}), {}".format(new, hard, e))
+ logger.exception("While setting RLIMIT_STACK")
+ arc_root = core.config.find_root(fail_on_error=False)
+ if arc_root is not None:
+ logger.debug('Arcadia root found: %s', arc_root)
+ extra_args = ['-ex', 'set substitute-path /-S/ {}/'.format(arc_root)] + extra_args
+ extra_args = ['-ex', 'set filename-display absolute'] + extra_args
+ if (
+ tool_name == 'arc'
+ and params.username not in {'sandbox', 'root'}
+ and os.getenv('YA_ALLOW_TOOL_ARC', 'no') != 'yes'
+ ):
+ message = (
+ 'Please, use natively installed arc, install guide:'
+ ' https://docs.yandex-team.ru/devtools/intro/quick-start-guide#arc-setup'
+ )
+ raise core.yarg.ArgsValidatingException(message)
+ exts.process.execve(tool_path, extra_args, env=env)
+
+ if lock_result:
+ lock_resource(toolchain_root(tool_name, params.toolchain, for_platform))
diff --git a/devtools/ya/handlers/tool/ya.make b/devtools/ya/handlers/tool/ya.make
new file mode 100644
index 0000000000..6021f3ea29
--- /dev/null
+++ b/devtools/ya/handlers/tool/ya.make
@@ -0,0 +1,33 @@
+PY23_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ NAMESPACE handlers.tool
+ __init__.py
+)
+
+PEERDIR(
+ devtools/ya/app
+ devtools/ya/build/build_opts
+ devtools/ya/core
+ devtools/ya/core/config
+ devtools/ya/core/respawn
+ devtools/ya/core/yarg
+ devtools/ya/exts
+ devtools/ya/yalibrary/platform_matcher
+ devtools/ya/yalibrary/tools
+ devtools/ya/yalibrary/toolscache
+ library/python/windows
+)
+
+END()
+
+RECURSE(
+ tests
+)
+
+RECURSE_FOR_TESTS(
+ bin
+ tests
+)
diff --git a/devtools/ya/handlers/ya.make b/devtools/ya/handlers/ya.make
new file mode 100644
index 0000000000..d88914356d
--- /dev/null
+++ b/devtools/ya/handlers/ya.make
@@ -0,0 +1,151 @@
+PY23_LIBRARY(ya-lib)
+
+SRCDIR(devtools/ya)
+
+PEERDIR(
+ devtools/ya/handlers/make
+ devtools/ya/handlers/package
+ devtools/ya/handlers/test
+ devtools/ya/handlers/tool
+ devtools/ya/handlers/ide
+ devtools/ya/handlers/dump
+ devtools/ya/handlers/gc
+ devtools/ya/handlers/gen_config
+ devtools/ya/handlers/maven_import
+ devtools/ya/handlers/java
+ # devtools/ya/handlers/analyze_make
+)
+
+IF (NOT YA_OPENSOURCE)
+ PEERDIR(
+ devtools/ya/handlers/__trace__
+ devtools/ya/handlers/addremove
+ devtools/ya/handlers/analyze_make
+ devtools/ya/handlers/buf
+ devtools/ya/handlers/check
+ devtools/ya/handlers/clang_tidy
+ devtools/ya/handlers/clone
+ devtools/ya/handlers/completion
+ devtools/ya/handlers/dctl
+ devtools/ya/handlers/download
+ devtools/ya/handlers/exec
+ devtools/ya/handlers/fetch
+ devtools/ya/handlers/fix_includes
+ devtools/ya/handlers/krevedko
+ devtools/ya/handlers/notify
+ devtools/ya/handlers/paste
+ devtools/ya/handlers/pr
+ devtools/ya/handlers/profile
+ devtools/ya/handlers/project
+ devtools/ya/handlers/py
+ devtools/ya/handlers/remote_gdb
+ devtools/ya/handlers/repo_check
+ devtools/ya/handlers/shell
+ devtools/ya/handlers/shelve
+ devtools/ya/handlers/stat
+ devtools/ya/handlers/style
+ devtools/ya/handlers/svn
+ devtools/ya/handlers/unshelve
+ devtools/ya/handlers/upload
+ devtools/ya/handlers/vmctl
+ devtools/ya/handlers/webide
+ devtools/ya/handlers/whoami
+ devtools/ya/handlers/wine
+ devtools/ya/handlers/yav
+
+ devtools/ya/handlers/py23migration/_ya0bin2
+ devtools/ya/handlers/py23migration/_ya0bin3
+ devtools/ya/handlers/py23migration/_ya2bin0
+ devtools/ya/handlers/py23migration/_ya2bin2
+ devtools/ya/handlers/py23migration/_ya3bin0
+ devtools/ya/handlers/py23migration/_ya3bin3
+ )
+ IF (PYTHON3)
+ PEERDIR(
+ devtools/ya/handlers/py23migration/py23_utils
+ devtools/ya/handlers/vim
+ devtools/ya/handlers/curl
+ devtools/ya/handlers/neovim
+ devtools/ya/handlers/gdb
+ devtools/ya/handlers/emacs
+ devtools/ya/handlers/grep
+ devtools/ya/handlers/jstyle
+ devtools/ya/handlers/nile
+ devtools/ya/handlers/sed
+ devtools/ya/handlers/ydb
+ devtools/ya/handlers/yql
+ )
+ ENDIF()
+ENDIF()
+END()
+
+RECURSE(
+ __trace__
+ addremove
+ analyze_make
+ autocheck
+ buf
+ check
+ clang_tidy
+ clone
+ completion
+ curl
+ dctl
+ download
+ dump
+ emacs
+ exec
+ fetch
+ fix_includes
+ gc
+ gdb
+ gen_config
+ grep
+ ide
+ java
+ jstyle
+ krevedko
+ make
+ maven_import
+ neovim
+ nile
+ notify
+ package
+ paste
+ pr
+ profile
+ project
+ py
+ remote_gdb
+ repo_check
+ sed
+ shell
+ shelve
+ stat
+ style
+ svn
+ test
+ tool
+ unshelve
+ upload
+ vim
+ vmctl
+ webide
+ whoami
+ wine
+ yav
+ ydb
+ yql
+)
+
+IF (NOT OPENSOURCE)
+ RECURSE(
+ py23migration/_ya0bin2
+ py23migration/_ya0bin3
+ py23migration/_ya2bin0
+ py23migration/_ya3bin0
+ py23migration/_ya3bin3
+ py23migration/_ya2bin2
+ py23migration/py23_utils
+ )
+ENDIF()
diff --git a/devtools/ya/opensource/ya.conf b/devtools/ya/opensource/ya.conf
new file mode 100644
index 0000000000..fb140d072a
--- /dev/null
+++ b/devtools/ya/opensource/ya.conf
@@ -0,0 +1,92 @@
+# Please keep this in sync with arcadia/ya.conf
+
+build_cache = true
+build_cache_conf = ['cas_logging=true', 'graph_info=true']
+build_cache_master = true
+cache_codec = ''
+cache_size = 150374182400
+content_uids = true
+dir_outputs = true
+dir_outputs_test_mode = true
+dump_debug_enabled = true
+fail_maven_export_with_tests = true
+incremental_build_dirs_cleanup = true
+oauth_exchange_ssh_keys = true
+remove_implicit_data_path = true
+remove_result_node = true
+tools_cache = true
+tools_cache_master = true
+use_atd_revisions_info = true
+use_jstyle_server = true
+use_command_file_in_testtool = true
+
+[test_tool3_handlers]
+build_clang_coverage_report = true
+build_go_coverage_report = true
+build_python_coverage_report = false
+build_sancov_coverage_report = true
+build_ts_coverage_report = true
+canonization_result_node = true
+canonize = true
+check_external = true
+check_mds = true
+check_resource = true
+checkout = true
+cov_merge_vfs = false
+create_allure_report = true
+download = true
+list_result_node = true
+list_tests = true
+merge_coverage_inplace = true
+merge_python_coverage = true
+minimize_fuzz_corpus = true
+populate_token_to_sandbox_vault = true
+resolve_clang_coverage = true
+resolve_go_coverage = true
+resolve_java_coverage = true
+resolve_python_coverage = false
+resolve_sancov_coverage = true
+resolve_ts_coverage = true
+result_node = true
+results_accumulator = true
+results_merger = true
+run_boost_test = true
+run_check = true
+run_clang_tidy = true
+run_classpath_clash = true
+run_coverage_extractor = true
+run_custom_lint = true
+run_diff_test = false
+run_eslint = true
+run_exectest = true
+run_fuzz = true
+run_fuzz_result_node = true
+run_g_benchmark = true
+run_go_fmt = true
+run_go_test = true
+run_go_vet = true
+run_gtest = true
+run_hermione = true
+run_javastyle = true
+run_jest = true
+run_ktlint_test = true
+run_pyimports = true
+run_skipped_test = true
+run_test = true
+run_ut = true
+run_y_benchmark = true
+sandbox_run_test = true
+unify_clang_coverage = true
+upload = false
+upload_coverage = false
+ytexec_run_test = true
+
+# ===== opensource only table params =====
+
+[host_platform_flags]
+OPENSOURCE = "yes"
+USE_PREBUILT_TOOLS = "no"
+
+[flags]
+OPENSOURCE = "yes"
+USE_PREBUILT_TOOLS = "no"