diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2023-12-02 01:45:21 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2023-12-02 02:42:50 +0300 |
commit | 9c43d58f75cf086b744cf4fe2ae180e8f37e4a0c (patch) | |
tree | 9f88a486917d371d099cd712efd91b4c122d209d /devtools/ya | |
parent | 32fb6dda1feb24f9ab69ece5df0cb9ec238ca5e6 (diff) | |
download | ydb-9c43d58f75cf086b744cf4fe2ae180e8f37e4a0c.tar.gz |
Intermediate changes
Diffstat (limited to 'devtools/ya')
30 files changed, 5080 insertions, 0 deletions
diff --git a/devtools/ya/chameleon_bin/__main__.py b/devtools/ya/chameleon_bin/__main__.py new file mode 100644 index 0000000000..f79b940fdb --- /dev/null +++ b/devtools/ya/chameleon_bin/__main__.py @@ -0,0 +1,59 @@ +import logging +import os +import shutil +from library.python.testing.recipe import declare_recipe +import yatest.common + + +class ChameleonRecipe: + def __init__(self): + self.bin2_root: str = None + self.bin3_root: str = None + self.bin2: str = None + self.bin3: str = None + + def post_init(self): + self.bin2_root = yatest.common.build_path("devtools/ya/bin") + self.bin3_root = yatest.common.build_path("devtools/ya/bin3") + self.bin2 = os.path.join(self.bin2_root, "ya-bin") + self.bin3 = os.path.join(self.bin3_root, "ya-bin") + + logging.info("ya-bin2 exists: %s", os.path.exists(self.bin2)) + logging.info("ya-bin3 exists: %s", os.path.exists(self.bin3)) + + @classmethod + def check_argv_py3(cls, argv): + return argv[0] == "yes" + + def start(self, argv): + self.post_init() + + logging.info("args: %s", argv) + + assert not (os.path.exists(self.bin2) and os.path.exists(self.bin3)), "Only one version of ya-bin can be in DEPENDS" + + if self.check_argv_py3(argv): + if not os.path.exists(self.bin2): + if not os.path.exists(self.bin2_root): + os.mkdir(self.bin2_root) + logging.debug("Create bin2 root folder: %s", self.bin2_root) + else: + logging.debug("Maybe test was restarted") + shutil.copy(self.bin3, self.bin2) + logging.debug("copy ya-bin3 to bin2 root: %s -> %s", self.bin3, self.bin2) + + + def stop(self, argv): + self.post_init() + + logging.info("args: %s", argv) + + if self.check_argv_py3(argv): + if os.path.exists(self.bin2): + logging.debug("Remove bin2 executable: %s", self.bin2) + # TOFIX v-korovin + os.remove(self.bin2) + +if __name__ == "__main__": + recipe = ChameleonRecipe() + declare_recipe(recipe.start, recipe.stop) diff --git a/devtools/ya/chameleon_bin/recipe.inc b/devtools/ya/chameleon_bin/recipe.inc new file mode 100644 index 0000000000..4e705cc996 --- /dev/null +++ b/devtools/ya/chameleon_bin/recipe.inc @@ -0,0 +1,32 @@ +DEPENDS(devtools/ya/chameleon_bin) + +INCLUDE(${ARCADIA_ROOT}/devtools/ya/opensource.inc) + +IF (PYTHON3) + DEPENDS(devtools/ya/bin3) +ELSE() + IF (NOT OPENSOURCE) + DEPENDS(devtools/ya/bin) + ENDIF() +ENDIF() + +IF (NEBIUS) + DATA( + arcadia/nebius/devtools/ya-bin/ya.conf + ) +ELSEIF (OPENSOURCE) + DATA( + arcadia/devtools/ya/opensource/ya.conf + ) +ELSE() + DATA( + arcadia/ya.conf + ) +ENDIF() + +DATA( + arcadia/ya +) + +USE_RECIPE(devtools/ya/chameleon_bin/chameleon_bin ${PYTHON3}) + diff --git a/devtools/ya/chameleon_bin/ya.make b/devtools/ya/chameleon_bin/ya.make new file mode 100644 index 0000000000..3b418c2651 --- /dev/null +++ b/devtools/ya/chameleon_bin/ya.make @@ -0,0 +1,12 @@ +PY3_PROGRAM() + +PY_SRCS(__main__.py) + +PEERDIR( + library/python/testing/recipe + library/python/testing/yatest_common +) + +NO_LINT() + +END() diff --git a/devtools/ya/handlers/dump/__init__.py b/devtools/ya/handlers/dump/__init__.py new file mode 100644 index 0000000000..77eb2f45d2 --- /dev/null +++ b/devtools/ya/handlers/dump/__init__.py @@ -0,0 +1,1161 @@ +from __future__ import absolute_import +from __future__ import print_function +import csv +import pickle +import exts.yjson as json +import logging +import os +import sys +import enum +import six + +from build.build_facade import ( + gen_all_loops, + gen_dir_loops, + gen_graph, + gen_licenses_list, + gen_forced_deps_list, + gen_modules, + gen_module_info, + gen_plan, + gen_json_graph, + gen_uids, + gen_srcdeps, + gen_dependencies, + gen_test_dart, + gen_conf, + gen_filelist, + gen_relation, +) +from handlers.dump.gen_conf_docs import dump_mmm_docs +from build.dir_graph import reachable, gen_dir_graph +from build.compilation_database import dump_compilation_database, COMPILATION_DATABASE_OPTS +from exts.strtobool import strtobool +from exts.tmp import temp_dir +from core.yarg import ( + CompositeHandler, + OptsHandler, + ArgConsumer, + FreeArgConsumer, + SetValueHook, + Options, + SetConstValueHook, + SetConstAppendHook, + UpdateValueHook, + SetAppendHook, + ExtendHook, + ArgsValidatingException, + Group, +) +from yalibrary.vcs import vcsversion +from core.imprint import imprint +from build.build_opts import ( + YMakeDebugOptions, + YMakeBinOptions, + FlagsOptions, + CustomFetcherOptions, + ToolsOptions, + SandboxAuthOptions, + JavaSpecificOptions, +) +from build.build_opts import ( + BuildTypeOptions, + BuildTargetsOptions, + ShowHelpOptions, + CustomBuildRootOptions, + ContinueOnFailOptions, +) +from build.build_opts import YMakeRetryOptions, ConfigurationPresetsOptions +from core.common_opts import CrossCompilationOptions, YaBin3Options +from test.explore import generate_tests_by_dart +from devtools.ya.test.dartfile import decode_recipe_cmdline + +import app +import app_config + +logger = logging.getLogger(__name__) + + +class DumpYaHandler(CompositeHandler): + common_opts = [ShowHelpOptions()] + + @staticmethod + def common_build_facade_opts(with_free_targets=True): + return [ + FlagsOptions(), + CustomFetcherOptions(), + SandboxAuthOptions(), + YMakeBinOptions(), + YMakeRetryOptions(), + CrossCompilationOptions(), + BuildTypeOptions('release'), + ContinueOnFailOptions(), + BuildTargetsOptions(with_free=with_free_targets), + ToolsOptions(), + ConfigurationPresetsOptions(), + YaBin3Options(), + ] + + def __init__(self): + CompositeHandler.__init__(self, description='Repository related information') + self['modules'] = OptsHandler( + action=app.execute(action=do_modules), + description='All modules', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DepTraverseOptions(), + DepFilterOptions(), + PeerDirectoriesOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + ], + ) + self['module-info'] = OptsHandler( + action=app.execute(action=do_module_info), + description='Modules info', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DepTraverseOptions(), + DepFilterOptions(), + PeerDirectoriesOptions(), + DataOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + DumpModuleInfoOptions(), + ], + ) + self['src-deps'] = OptsHandler( + action=app.execute(action=do_dump_srcdeps), + description='Dump of all source dependencies', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DumpSrcDepsOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + ], + ) + self['dir-graph'] = OptsHandler( + action=app.execute(action=do_dir_graph), + description='Dependencies between directories', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DumpDirOptions(), + SplitByTypeOptions(), + DumpTestListOptions(), + DepTraverseOptions(), + DepFilterOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + LegacyDepsOptions(), + ], + ) + self['dep-graph'] = OptsHandler( + action=app.execute(action=do_dep_graph), + description='Dependency internal graph', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DepGraphOutputOptions(), + DepTraverseOptions(), + DepFilterOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + LegacyDepsOptions(), + ], + ) + self['dot-graph'] = OptsHandler( + action=app.execute(action=do_dot_graph), + description='Dependency between directories in dot format', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DepTraverseOptions(), + DepFilterOptions(), + PeerDirectoriesOptions(), + CustomBuildRootOptions(), + ], + ) + self['json-dep-graph'] = OptsHandler( + action=app.execute(action=do_json_dep_graph), + description='Dependency graph as json', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DepTraverseOptions(), + DepFilterOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + LegacyDepsOptions(), + ], + ) + self['build-plan'] = OptsHandler( + action=app.execute(action=do_build_plan), + description='Build plan', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DepTraverseOptions(), + CustomBuildRootOptions(), + YMakeDebugOptions(), + ], + ) + self['compile-commands'] = OptsHandler( + action=app.execute(action=dump_compilation_database), + description='JSON compilation database', + opts=COMPILATION_DATABASE_OPTS + [ToolsOptions(), CustomFetcherOptions(), SandboxAuthOptions()], + ) + self['compilation-database'] = OptsHandler( + action=app.execute(action=dump_compilation_database), + description='Alias for compile-commands', + opts=COMPILATION_DATABASE_OPTS + [ToolsOptions(), CustomFetcherOptions(), SandboxAuthOptions()], + ) + self['relation'] = OptsHandler( + action=app.execute(action=do_relation), + description='PEERDIR relations. Please don\'t run from the arcadia root.', + opts=self.common_opts + + self.common_build_facade_opts(False) + + [ + RelationSrcDstOptions(free_dst=True), + DepTraverseOptions(), + DepFilterOptions(), + PeerDirectoriesOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + ], + ) + self['all-relations'] = OptsHandler( + action=app.execute(action=do_all_relations), + description='All relations between internal graph nodes in dot format. Please don\'t run from the arcadia root.', + opts=self.common_opts + + self.common_build_facade_opts(False) + + [ + RelationSrcDstOptions(free_dst=True), + DumpAllRelationsOptions(), + DepTraverseOptions(), + DepFilterOptions(), + PeerDirectoriesOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + ], + ) + self['files'] = OptsHandler( + action=app.execute(action=do_files), + description='File list', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + FilesFilterOptions(), + DepTraverseOptions(), + DepFilterOptions(), + DataOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + ], + ) + self['loops'] = OptsHandler( + action=app.execute(action=do_loops), + description='All loops in arcadia', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DepTraverseOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + ], + ) + self['peerdir-loops'] = OptsHandler( + action=app.execute(action=do_peerdir_loops), + description='Loops by peerdirs', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + DepTraverseOptions(), + YMakeDebugOptions(), + CustomBuildRootOptions(), + ], + ) + self['imprint'] = OptsHandler( + action=app.execute(action=do_imprint), + description='Directory imprint', + opts=self.common_opts + [CustomFetcherOptions(), SandboxAuthOptions(), ToolsOptions()], + ) + self['uids'] = OptsHandler( + action=app.execute(action=do_uids), + description='All targets uids', + opts=self.common_opts + self.common_build_facade_opts(), + visible=False, + ) + self['test-list'] = OptsHandler( + action=app.execute(action=do_test_list), + description='All test entries', + opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions()], + ) + self['json-test-list'] = OptsHandler( + action=app.execute(action=do_json_test_list), + description='All test entries as json', + opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions()], + ) + self['conf'] = OptsHandler( + action=app.execute(action=do_conf), + description='Print build conf', + opts=self.common_opts + + [ + FlagsOptions(), + CustomFetcherOptions(), + SandboxAuthOptions(), + CrossCompilationOptions(), + ToolsOptions(), + BuildTypeOptions('release'), + JavaSpecificOptions(), + ], + visible=False, + ) + self['licenses'] = OptsHandler( + action=app.execute(action=do_licenses), + description='Print known licenses grouped by their properties', + opts=self.common_opts + + self.common_build_facade_opts() + + [ + YMakeDebugOptions(), + CustomBuildRootOptions(), + DumpLicenseOptions(), + ], + visible=False, + ) + self['forced-deps'] = OptsHandler( + action=app.execute(action=do_forced_deps), + description='Print known forced dependency management', + opts=FullForcedDepsOptions(), + visible=False, + ) + self['conf-docs'] = OptsHandler( + action=app.execute(action=do_conf_docs), + description='Print descriptions of entities (modules, macros, multimodules, etc.)', + opts=self.common_opts + self.common_build_facade_opts() + [DumpDescriptionOptions()], + ) + self['root'] = OptsHandler( + action=app.execute(lambda params: sys.stdout.write(params.arc_root) and 0), + description='Print Arcadia root', + opts=self.common_opts, + ) + self['vcs-info'] = OptsHandler( + action=app.execute(action=do_dump_vcs_info), + description='Print VCS revision information.', + opts=self.common_opts + self.common_build_facade_opts(), + visible=False, + ) + self['raw-vcs-info'] = OptsHandler( + action=app.execute(action=do_dump_raw_vcs_info), + description='Print VCS revision information.', + opts=self.common_opts + self.common_build_facade_opts(), + visible=False, + ) + self['svn-revision'] = OptsHandler( + action=app.execute(action=do_dump_svn_revision), + description='Print SVN revision information.', + opts=self.common_opts + self.common_build_facade_opts(), + visible=False, + ) + self['recipes'] = OptsHandler( + action=app.execute(action=do_recipes), + description='All recipes used in tests', + opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions(), DumpRecipesOptions()], + ) + if app_config.in_house: + import devtools.ya.handlers.dump.arcadia_specific as arcadia_specific + from handlers.dump.debug import debug_handler + + self['groups'] = arcadia_specific.GroupsHandler() + self['atd-revisions'] = OptsHandler( + action=app.execute(action=arcadia_specific.do_atd_revisions), + description='Dump revisions of trunk/arcadia_tests_data', + opts=self.common_opts + + [DumpAtdRevisionOptions(), CustomFetcherOptions(), SandboxAuthOptions(), ToolsOptions()], + ) + self['debug'] = debug_handler + + +def _do_dump(gen_func, params, debug_options=[], write_stdout=True, build_root=None, **kwargs): + for name in 'debug_options', 'filter_opts', 'peerdir_opts': + if hasattr(params, name): + debug_options.extend(getattr(params, name)) + logger.debug('abs_targets: %s', params.abs_targets) + with temp_dir() as tmp: + res = gen_func( + build_root=build_root or tmp, + build_type=params.build_type, + build_targets=params.abs_targets, + debug_options=debug_options, + flags=params.flags, + ymake_bin=getattr(params, 'ymake_bin', None), + host_platform=params.host_platform, + target_platforms=params.target_platforms, + **kwargs + ) + if write_stdout: + sys.stdout.write(res.stdout) + return res + + +def do_modules(params): + _do_dump(gen_modules, params) + + +def do_module_info(params, write_stdout=True): + if params.with_data: + params.flags['YMAKE_ADD_DATA'] = 'yes' + return _do_dump( + gen_func=gen_module_info, + params=params, + modules_info_filter=getattr(params, 'modules_info_filter'), + modules_info_file=getattr(params, 'modules_info_file'), + write_stdout=write_stdout, + ) + + +def do_dir_graph(params): + def merge_lists(one, two): + return sorted(set(one + two)) + + def get_test_dependencies(deps, arc_root, test, test_data=None): + data = merge_lists(test_data or [], deps) + return merge_lists(data, get_canondata_paths(arc_root, test)) + + if params.dump_deps: + deps = _do_dump(gen_dependencies, params, params.legacy_deps_opts, write_stdout=False) + json.dump(json.loads(deps.stdout), sys.stdout, indent=4, sort_keys=True) + return + + if params.dump_reachable_dirs: + _do_dump(gen_graph, params, params.legacy_deps_opts + ['x', 'M']) + return + + dg = _do_dump( + gen_dir_graph, params, params.legacy_deps_opts, write_stdout=False, split_by_types=params.split_by_type + ) + + tests = get_tests(params) + for test in tests: + test_data = test.get_arcadia_test_data() + if test.project_path in test_data: + test_data.remove(test.project_path) + if params.split_by_type: + if test.project_path not in dg: + dg[test.project_path] = {} + dg[test.project_path]["INCLUDE"] = get_test_dependencies( + dg[test.project_path].get("INCLUDE", []), params.arc_root, test + ) + dg[test.project_path]["DATA"] = merge_lists(dg[test.project_path].get("DATA", []), test_data) + else: + dg[test.project_path] = get_test_dependencies( + dg.get(test.project_path, []), params.arc_root, test, test_data + ) + + if params.trace_from is not None: + print('\n'.join(reachable(dg, params.trace_from, params.split_by_type))) + + elif params.plain_dump: + + def get_plain_deps(): + plain_deps = set() + for k, v in six.iteritems(dg): + plain_deps.add(k) + plain_deps |= set(v) + return sorted(list(plain_deps)) + + json.dump(get_plain_deps(), sys.stdout, indent=4, sort_keys=True) + else: + json.dump(dg, sys.stdout, indent=4, sort_keys=True) + + +def get_canondata_paths(arc_root, test): + paths = [] + canondata_path = os.path.join(arc_root, test.project_path, "canondata") + if os.path.exists(canondata_path): + for root, dirs, files in os.walk(canondata_path): + paths.append(os.path.relpath(root, arc_root)) + return paths + + +DUMP_OPTS_GROUP = Group("Dump options", 0) + + +class SplitByTypeOptions(Options): + def __init__(self): + self.split_by_type = False + + @staticmethod + def consumer(): + return ArgConsumer( + ['--split'], + help='split by type', + hook=SetConstValueHook('split_by_type', True), + group=DUMP_OPTS_GROUP, + ) + + +class DumpDirOptions(Options): + def __init__(self): + self.trace_from = None + self.plain_dump = False + self.dump_deps = False + self.dump_reachable_dirs = False + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--trace'], + help='trace from sources', + hook=SetValueHook('trace_from'), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--plain'], + help='plain dump mode', + hook=SetConstValueHook('plain_dump', True), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--dump-deps'], + help='Dump dependencies (legacy)', + hook=SetConstValueHook('dump_deps', True), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--reachable-dirs'], + help='Dump reachable dirs', + hook=SetConstValueHook('dump_reachable_dirs', True), + group=DUMP_OPTS_GROUP, + ), + ] + + +class PeerDirectoriesOptions(Options): + def __init__(self): + self.peerdir_opts = [] + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--old-peerdirs'], + help='Include indirect (module->dir->module) PEERDIRs', + hook=SetConstAppendHook('peerdir_opts', 'I'), + group=DUMP_OPTS_GROUP, + visible=False, + ) + ] + + +class LegacyDepsOptions(Options): + def __init__(self): + self.legacy_deps_opts = ['I'] + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--no-legacy-deps'], + help='Exclude legacy dependencies (inderect PEERDIR relations)', + hook=SetConstValueHook('legacy_deps_opts', []), + group=DUMP_OPTS_GROUP, + ) + ] + + +class DumpSrcDepsOptions(Options): + def __init__(self): + self.with_yamake = False + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--with-yamakes'], + help='Include ya.make files, that are only used as a build configuration', + hook=SetConstValueHook('with_yamake', True), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DepFilterOptions(Options): + def __init__(self): + self.filter_opts = [] + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--no-tools'], + help='Exclude tools dependencies', + hook=SetConstAppendHook('filter_opts', 'V'), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--no-addincls'], + help='Exclude ADDINCLs dependencies', + hook=SetConstAppendHook('filter_opts', 'T'), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DepGraphOutputType(enum.Enum): + CUSTOM = 0 + FLAT_JSON = 1 + FLAT_JSON_FILES = 2 + + +class DepGraphOutputOptions(Options): + def __init__(self): + self.output_type = DepGraphOutputType.CUSTOM + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--flat-json'], + help='Dump dep-graph in flat json format', + hook=SetConstValueHook('output_type', DepGraphOutputType.FLAT_JSON), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--flat-json-files'], + help='Dump dep-graph in flat json format without commands', + hook=SetConstValueHook('output_type', DepGraphOutputType.FLAT_JSON_FILES), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DepTraverseOptions(Options): + @staticmethod + def consumer(): + def test_depends_updater(flags): + flags['TRAVERSE_DEPENDS'] = 'yes' + flags['TRAVERSE_RECURSE_FOR_TESTS'] = 'yes' + return flags + + def ignore_recurses_updater(flags): + flags['TRAVERSE_RECURSE'] = 'no' + return flags + + # Hooks update flags property from FlagsOptions + return [ + ArgConsumer( + ['-t', '--force-build-depends'], + help='Include DEPENDS and RECURSE_FOR_TESTS dependencies', + hook=UpdateValueHook('flags', test_depends_updater), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['-A'], + help='Same as -t', + hook=UpdateValueHook('flags', test_depends_updater), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--ignore-recurses'], + help='Exclude all RECURSE dependencies', + hook=UpdateValueHook('flags', ignore_recurses_updater), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DataOptions(Options): + def __init__(self): + self.with_data = False + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--with-data'], + help='Include DATA files and dirs', + hook=SetConstValueHook('with_data', True), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DumpTestListOptions(Options): + def __init__(self): + self.follow_dependencies = False + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--skip-deps'], + help='Now default and ignored', + hook=SetConstValueHook('follow_dependencies', False), + group=DUMP_OPTS_GROUP, + visible=False, + deprecated=True, + ), + ] + + +class DumpLicenseOptions(Options): + def __init__(self): + self.json_licenses = False + self.link_type = 'static' + self.custom_tags = [] + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--json'], + help='Machine oriented json representation', + hook=SetConstValueHook('json_licenses', True), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--link-type'], + help='Specify library link type (static|dynamic)', + hook=SetValueHook('link_type'), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--custom-license-tag'], + help='Specify library link type (static|dynamic)', + hook=SetAppendHook('custom_tags'), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DumpForcedDepsOptions(Options): + def __init__(self): + self.json_forced_deps = False + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--json'], + help='Json forced dependency representation', + hook=SetConstValueHook('json_forced_deps', True), + group=DUMP_OPTS_GROUP, + ), + ] + + +def FullForcedDepsOptions(): + return ( + DumpYaHandler.common_opts + + DumpYaHandler.common_build_facade_opts() + + [ + YMakeDebugOptions(), + CustomBuildRootOptions(), + DumpForcedDepsOptions(), + ] + ) + + +class RelationSrcDstOptions(Options): + def __init__(self, free_dst=False): + self._free_dst = free_dst + self.recursive = False + self.relation_src = [] + self.relation_dst = [] + + def consumer(self): + res = [ + ArgConsumer( + ['--from'], + help='Dump relations from this target (path relative to the arcadia root)', + hook=SetAppendHook('relation_src'), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--to'], + help='Dump relations to this target (path relative to the arcadia root)', + hook=SetAppendHook('relation_dst'), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--recursive'], + help='Show relations between RELATION_SRC and all modules from RELATION_DST directories', + hook=SetConstValueHook('recursive', True), + group=DUMP_OPTS_GROUP, + ), + ] + if self._free_dst: + res.append( + FreeArgConsumer( + help='relation destination target', + hook=ExtendHook('relation_dst'), + ) + ) + return res + + def postprocess(self): + if len(self.relation_dst) == 0: + raise ArgsValidatingException('Error: no target is set') + + +class FilesFilterOptions(Options): + def __init__(self): + self.skip_make_files = False + self.mark_make_files = False + + def consumer(self): + res = [ + ArgConsumer( + ['--skip-make-files'], + help='Skip all make files', + hook=SetConstValueHook('skip_make_files', True), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--mark-make-files'], + help='Mark all make files as Makefile', + hook=SetConstValueHook('mark_make_files', True), + group=DUMP_OPTS_GROUP, + ), + ] + return res + + +class DumpAllRelationsOptions(Options): + def __init__(self): + self.json_format = False + self.show_targets_deps = False + + def consumer(self): + res = [ + ArgConsumer( + ['--json'], + help='Dump relations in json format', + hook=SetConstValueHook('json_format', True), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--show-targets-deps'], + help='Show dependencies between RELATION_DST targets', + hook=SetConstValueHook('show_targets_deps', True), + group=DUMP_OPTS_GROUP, + ), + ] + return res + + +class DumpDescriptionOptions(Options): + def __init__(self): + self.dump_all_conf_docs = False + self.conf_docs_json = False + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--dump-all'], + help='Dump information for all entities including internal', + hook=SetConstValueHook('dump_all_conf_docs', True), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--json'], + help='Dump information for all entities including internal in json format, uses are included', + hook=SetConstValueHook('conf_docs_json', True), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DumpAtdRevisionOptions(Options): + def __init__(self): + self.human_readable = False + self.print_size = False + self.arcadia_revision = None + self.path = None + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--human-readable'], + help='Add human-readable paths to comments', + hook=SetConstValueHook('human_readable', True), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--print-size'], + help='Add human-readable paths to comments', + hook=SetConstValueHook('print_size', True), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--revision'], + help='Gather revisions for this arcadia revision', + hook=SetValueHook('arcadia_revision'), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--path'], + help='Gather revisions for this path recursively', + hook=SetValueHook('path'), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DumpModuleInfoOptions(Options): + def __init__(self): + self.modules_info_file = '' + self.modules_info_filter = '' + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--modules-info-file'], + help='Save modules informaton into file specified', + hook=SetValueHook('modules_info_file'), + group=DUMP_OPTS_GROUP, + ), + ArgConsumer( + ['--modules-info-filter'], + help='Dump information only for modules matching regexp', + hook=SetValueHook('modules_info_filter'), + group=DUMP_OPTS_GROUP, + ), + ] + + +class DumpRecipesOptions(Options): + def __init__(self): + self.json_format = False + + def consumer(self): + res = [ + ArgConsumer( + ['--json'], + help='Dump recipes in json format', + hook=SetConstValueHook('json_format', True), + group=DUMP_OPTS_GROUP, + ), + ] + return res + + +def do_dump_srcdeps(params): + debug_options = [] + params.flags['YMAKE_ADD_DATA'] = 'yes' + if params.with_yamake: + debug_options.append('mkf') + _do_dump(gen_srcdeps, params, debug_options) + + +def do_dot_graph(params): + _do_dump(gen_modules, params, ['D']) + + +def do_dep_graph(params): + options = params.legacy_deps_opts + if params.output_type == DepGraphOutputType.FLAT_JSON: + options.append('flat-json-with-cmds') + elif params.output_type == DepGraphOutputType.FLAT_JSON_FILES: + options.append('flat-json') + _do_dump(gen_graph, params, options) + + +def do_json_dep_graph(params): + _do_dump(gen_json_graph, params, params.legacy_deps_opts) + + +def do_licenses(params): + _do_dump( + gen_licenses_list, + params, + write_stdout=True, + debug_options=['lic-json' if params.json_licenses else 'lic'], + lic_link_type=params.link_type, + lic_custom_tags=params.custom_tags, + ) + + +def do_forced_deps(params, write_stdout=True): + res = _do_dump( + gen_forced_deps_list, + params, + write_stdout=write_stdout, + debug_options=['fdm-json' if params.json_forced_deps else 'fdm'], + ) + return '' if write_stdout else res.stdout + + +# TODO: support host/target platforms opts +def do_build_plan(params): + with temp_dir() as tmp: + graph = gen_plan( + arc_root=params.arc_root, + build_root=tmp, + build_type=params.build_type, + build_targets=params.abs_targets, + debug_options=params.debug_options, + flags=params.flags, + ymake_bin=params.ymake_bin, + no_ymake_resource=params.no_ymake_resource, + vcs_file=params.vcs_file, + ) + json.dump(graph, sys.stdout, indent=4, sort_keys=True) + + +def do_relation(params): + options = [] + if params.recursive: + options.append('recursive') + for dst in params.relation_dst: + _do_dump( + gen_relation, + params, + options, + find_path_from=params.relation_src, + find_path_to=[dst], + ) + + +def do_all_relations(params): + options = ['J' if params.json_format else 'D'] + if params.recursive: + options.append('recursive') + if params.show_targets_deps: + options.append('show-targets-deps') + _do_dump( + gen_relation, + params, + options, + find_path_from=params.relation_src, + find_path_to=params.relation_dst, + ) + + +def do_files(params): + options = [] + if params.with_data: + options = ['dump-data'] + params.flags['YMAKE_ADD_DATA'] = 'yes' + if params.skip_make_files: + options.append('skip-make-files') + if params.mark_make_files: + options.append('mark-make-files') + _do_dump(gen_filelist, params, options) + + +def do_loops(params): + _do_dump(gen_all_loops, params) + + +def do_peerdir_loops(params): + _do_dump(gen_dir_loops, params) + + +def do_imprint(params): + lines = imprint.generate_detailed_imprints(os.curdir) + writer = csv.writer(sys.stdout, delimiter='\t') + for line in lines: + writer.writerow(line) + + +# TODO: support host/target platforms opts +def do_uids(params): + uids = gen_uids( + arc_root=params.arc_root, + build_root=None, + build_type=params.build_type, + build_targets=params.abs_targets, + debug_options=[], + flags={}, + ymake_bin=None, + ) + json.dump(uids, sys.stdout, indent=4, sort_keys=True) + + +def get_tests(params): + params.flags['TRAVERSE_RECURSE_FOR_TESTS'] = 'yes' + kwargs = { + 'build_root': params.bld_root, + 'build_type': params.build_type, + 'abs_targets': params.abs_targets, + 'debug_options': [], + 'flags': params.flags, + 'ymake_bin': params.ymake_bin, + 'arcadia_tests_data_path': 'arcadia_tests_data', + } + ( + _, + test_dart, + ) = gen_test_dart(**kwargs) + + return sorted( + generate_tests_by_dart( + test_dart, + opts=params, + ), + key=lambda test: os.path.join(test.project_path, test.name), + ) + + +def do_json_test_list(params): + json.dump([t.save() for t in get_tests(params)], sys.stdout, indent=4, sort_keys=True) + + +def do_test_list(params): + print(pickle.dumps(get_tests(params))) + + +def do_conf(params): + with temp_dir() as tmp: + generation_conf = _do_dump(gen_conf, params, write_stdout=False, build_root=tmp) + print(open(generation_conf, 'r').read()) + + +def do_conf_docs(params): + _do_dump(dump_mmm_docs, params, dump_all_conf_docs=params.dump_all_conf_docs, conf_docs_json=params.conf_docs_json) + + +def do_dump_raw_vcs_info(params): + sys.stdout.write(json.dumps(vcsversion.get_raw_version_info(params.arc_root, params.bld_root))) + + +def do_dump_vcs_info(params): + fake_data = strtobool(params.flags.get('NO_VCS_DEPENDS', 'no')) + fake_build_info = strtobool(params.flags.get('CONSISTENT_BUILD', 'no')) + sys.stdout.write(vcsversion.get_version_info(params.arc_root, params.bld_root, fake_data, fake_build_info) + '\n') + + +def do_dump_svn_revision(params): + sys.stdout.write(str(vcsversion.repo_config(params.arc_root))) + + +def do_recipes(params): + tests = get_tests(params) + test_recipes = [] + for test in tests: + encoded_recipes = test.recipes + if not encoded_recipes: + continue + decoded_recipes = decode_recipe_cmdline(encoded_recipes) + test_recipes.append(decoded_recipes) + + recipes = set() + for test_recipe in test_recipes: + for recipe in test_recipe: + recipes.add(tuple(recipe)) + + if params.json_format: + output = json.dumps(recipes, indent=4, sort_keys=True) + else: + recipes_str = [' '.join(recipe) for recipe in recipes] + output = ' '.join(recipes_str) + print(output) diff --git a/devtools/ya/handlers/dump/gen_conf_docs.py b/devtools/ya/handlers/dump/gen_conf_docs.py new file mode 100644 index 0000000000..4b9978f226 --- /dev/null +++ b/devtools/ya/handlers/dump/gen_conf_docs.py @@ -0,0 +1,384 @@ +from __future__ import absolute_import, unicode_literals +import exts.yjson as json +import re +import os +import six + +import core.config + +import build.genconf +import build.ymake2 + +from yalibrary.vcs import vcsversion +from build.build_facade import gen_conf +from exts.strtobool import strtobool + + +class _Markdown: + header = '#' + alink = 'https://a.yandex-team.ru/arc/trunk/arcadia/' + # Section nesting count + scount = 2 + # Description nesting count + dcount = 6 + # Avoid too many entries in toc for macros + chunks_in_toc = 10 + # Checks description + internal_pattern = re.compile(r'^\s*@?internal[.]?\s*$', flags=re.IGNORECASE) + # Checks description + deprecated_pattern = re.compile(r'^\s*@?deprecated[.]?\s*$', flags=re.IGNORECASE) + # Checks header + h_patterns = { + 'internal': re.compile('.*#.*internal.*', flags=re.IGNORECASE), + 'deprecated': re.compile('.*#.*deprecated.*', flags=re.IGNORECASE), + } + + def __init__(self, arc_root, dump_all_descs, use_svn): + self.descs = {'macros': {}, 'modules': {}, 'multimodules': {}, 'unknowns': {}} + self.links = {} + self.anchors = {} + try: + self.svn_revision, _ = vcsversion.repo_config(arc_root) if use_svn else (-1, '') + except Exception: + self.svn_revision = -1 + self.dump_all_descs = dump_all_descs + + def process_entry(self, file, entry): + if 'kind' not in entry or entry['kind'] != 'node': + return + + self._add_entry_optionally(file, entry) + + def dump(self): + res = self._dump_toc() + + for type in ['multimodules', 'modules', 'macros', 'unknowns']: + if self.descs[type]: + res += _Markdown._format_section(type) + + for name in sorted(self.descs[type]): + for d in self.descs[type][name]['text']: + res += six.ensure_str(d) + + for type in ['multimodules', 'modules', 'macros', 'unknowns']: + if self.descs[type]: + for name in sorted(self.descs[type]): + link = self.descs[type][name]['link'] + res += six.ensure_str(link) + + return res + + def _add_entry_optionally(self, file, entry): + props = entry['properties'] + doc = {'name': props['name'], 'type': props['type'], 'file': file, 'line': entry['range']['line']} + + if 'comment' in props: + doc['long desc'] = props['comment'] + if 'usage' in props: + doc['usage'] = props['usage'] + + doc['revision'] = self.svn_revision + + descs, link = self._format_entry(doc) + + dictionary = ( + self.descs[doc['type'] + 's'] + if doc['type'] in ['macro', 'module', 'multimodule'] + else self.descs['unknowns'] + ) + + if not self.dump_all_descs and _Markdown._is_internal(descs[0]): + return + + dictionary[doc['name']] = {'text': descs, 'link': link, 'src_data': doc} + + @classmethod + def _is_internal(cls, header): + return cls.h_patterns['internal'].match(header) + + def _dump_toc(self): + res = '*Do not edit, this file is generated from comments to macros definitions using `ya dump conf-docs{all}`.*\n\n'.format( + all=' --dump-all' if self.dump_all_descs else '' + ) + res += '{markup} ya.make {all}commands\n\n'.format( + markup=_Markdown.header, all='and core.conf ' if self.dump_all_descs else '' + ) + res += ( + 'General info: [How to write ya.make files](https://wiki.yandex-team.ru/yatool/HowToWriteYaMakeFiles)\n\n' + ) + res += '{markup} Table of contents\n\n'.format(markup=_Markdown.header * 2) + + for type in ['multimodules', 'modules', 'macros', 'unknowns']: + if self.descs[type]: + res += _Markdown._format_toc_section(type) + if type != 'macros': + for name in sorted(self.descs[type]): + res += _Markdown._format_toc_header(self.descs[type][name]['src_data']) + else: + chunk_cnt = 0 + first_macro = {} + last_macro = {} + for name in sorted(self.descs[type]): + if chunk_cnt == 0: + first_macro = self.descs[type][name]['src_data'] + chunk_cnt += 1 + last_macro = self.descs[type][name]['src_data'] + + if chunk_cnt == _Markdown.chunks_in_toc: + chunk_cnt = 0 + res += _Markdown._format_toc_macro_header(first_macro, last_macro) + + if chunk_cnt != 0: + res += _Markdown._format_toc_macro_header(first_macro, last_macro) + return res + + @classmethod + def _format_entry(cls, doc): + descs = [] + + lines, special_tags = _Markdown._format_desc(doc) + descs.append(_Markdown._format_header(doc, special_tags)) + descs.append(lines) + return (descs, _Markdown._format_link(doc)) + + @classmethod + def _format_toc_section(cls, type): + return '{indent} * [{section}](#{anchor})\n'.format( + indent=' ' * cls.scount, section=type.capitalize(), anchor=type + ) + + @classmethod + def _format_section(cls, type): + return '{markup} {text} <a name="{anchor}"></a>\n\n'.format( + markup=cls.header * cls.scount, text=type.capitalize(), anchor=type + ) + + @staticmethod + def _format_header_anchor(doc): + return '<a name="{atype}_{aname}"></a>'.format(atype=doc['type'], aname=doc['name']) + + @classmethod + def _format_toc_header(cls, doc): + qual = doc['type'].capitalize() if doc['type'] in ['macro', 'module', 'multimodule'] else "Unknown" + return '{indent} - {qual} [{name}](#{type}_{name})\n'.format( + indent=' ' * cls.dcount, qual=qual, name=doc['name'], type=doc['type'] + ) + + @classmethod + def _format_toc_macro_header(cls, fdoc, ldoc): + return '{indent} - Macros [{fname}](#{ftype}_{fname}) .. [{lname}](#{ltype}_{lname})\n'.format( + indent=' ' * cls.dcount, fname=fdoc['name'], ftype=fdoc['type'], lname=ldoc['name'], ltype=ldoc['type'] + ) + + # Also adds special tags 'internal' and 'deprecated' + @classmethod + def _format_header(cls, doc, special_tags): + name = doc['name'] + usage = doc['usage'] if 'usage' in doc else "" + usage = name if not usage else usage + + qual = doc['type'].capitalize() if doc['type'] in ['macro', 'module', 'multimodule'] else "Unknown" + + usage = _Markdown._remove_formatting(usage.rstrip().lstrip()) + name = _Markdown._remove_formatting(name) + + usage = usage.replace(name, '[' + name + "][]", 1) + + if special_tags: + special = '' + for tag in special_tags: + if not cls.h_patterns[tag].match(usage): + special += ' ' + tag + # Emphasis + if usage.find("#") != -1: + usage += special + else: + usage += ' #' + special[1:] + + # Emphasis + if usage.find("#") != -1: + usage = usage.replace('#', "_#", 1) + usage += "_" + + return '{markup} {type} {usage} {anchor}\n'.format( + markup=cls.header * cls.dcount, type=qual, usage=usage, anchor=_Markdown._format_header_anchor(doc) + ) + + # Prints verbatim. Strips unnecessary indent and escapes '_'/'*'. + @classmethod + def _format_desc(cls, doc): + result = "" + + desc = "" + if 'long desc' in doc: + desc = doc['long desc'].rstrip() + if not desc: + desc = " Not documented yet.\n" + + lines = _Markdown._strip_blanks(desc.splitlines()) + lines = _Markdown._remove_formatting_markdown(lines) + + result += '\n'.join(lines) + "\n\n" + + special_tags = [] + if doc['name'].startswith('_') or any([cls.internal_pattern.match(x) for x in lines]): + special_tags.append("internal") + if any([cls.deprecated_pattern.match(x) for x in lines]): + special_tags.append("deprecated") + + return (result, special_tags) + + @staticmethod + def _format_link(doc): + return ' [{tag_name}]: {baselink}{file}{rev}#L{line}\n'.format( + tag_name=_Markdown._remove_formatting(doc['name']), + baselink=_Markdown.alink, + file=doc['file'], + rev='?rev=' + str(doc['revision']) if doc['revision'] > 0 else '', + line=doc['line'], + ) + + @staticmethod + def _strip_blanks(lines): + first = 0 + for line in lines: + if not line.lstrip().rstrip(): + first += 1 + else: + break + last = 0 + for line in reversed(lines): + if not line.lstrip().rstrip(): + last += 1 + else: + break + lines = lines[first : (len(lines) - last)] + lines = [x.rstrip().expandtabs(4) for x in lines] + indent = 10000 + for line in lines: + if line: + indent = min(indent, len(line) - len(line.lstrip())) + + if indent > 0: + lines = [x.replace(' ' * indent, '', 1) for x in lines] + + return lines + + # Conditionally removed formatting. + # Code blocks are not modified + @staticmethod + def _remove_formatting_markdown(lines): + code = False + new_paragraph = True + backtick_block = False + + res = [] + for line in lines: + if new_paragraph: + if line.startswith('\t') or line.startswith(' ' * 4): + code = True + + if not line.startswith('\t') and not line.startswith(' ' * 4): + code = False + + new_paragraph = not line + + if line.lstrip() == '```': + backtick_block = not backtick_block + + res.append(line if code or backtick_block else _Markdown._remove_formatting(line)) + return res + + # Unconditionally removes formatting due to '_'/'*' + @staticmethod + def _remove_formatting(x): + return x.replace("_", r"\_").replace("*", r"\*") + + +def _gen( + custom_build_directory, + build_type, + build_targets, + debug_options, + flags=None, + warn_mode=None, + ymake_bin=None, + platform=None, + host_platform=None, + target_platforms=None, + **kwargs +): + generation_conf = gen_conf( + build_root=custom_build_directory, + build_type=build_type, + build_targets=build_targets, + flags=flags, + host_platform=host_platform, + target_platforms=target_platforms, + ) + res, evlog_dump = build.ymake2.ymake_dump( + custom_build_directory=custom_build_directory, + build_type=build_type, + abs_targets=build_targets, + debug_options=debug_options, + warn_mode=warn_mode, + flags=flags, + ymake_bin=ymake_bin, + platform=platform, + grab_stderr=True, + custom_conf=generation_conf, + **kwargs + ) + return res + + +def dump_mmm_docs( + build_root, + build_type, + build_targets, + debug_options, + flags, + dump_all_conf_docs=None, + conf_docs_json=None, + ymake_bin=None, + platform=None, + host_platform=None, + target_platforms=None, +): + json_dump_name = os.path.join(build_root, 'ymake.dump.ydx.json') + arc_root = core.config.find_root_from(build_targets) + null_ya_make = os.path.join(arc_root, 'build', 'docs', 'empty') + + if not conf_docs_json: + if not os.path.exists(null_ya_make): + raise "Empty project not found, dump conf-docs may work too long" + + res = _gen( + custom_build_directory=build_root, + build_type=build_type, + # Override target + build_targets=[null_ya_make] if not conf_docs_json else build_targets, + debug_options=debug_options, + flags=flags, + ymake_bin=ymake_bin, + platform=platform, + host_platform=host_platform, + target_platforms=target_platforms, + yndex_file=json_dump_name, + ) + + if conf_docs_json: + with open(json_dump_name, 'r') as jfile: + res.stdout += jfile.read() + else: + with open(json_dump_name, 'r') as jfile: + contents = jfile.read() + jdata = json.loads(contents) + no_svn = True if 'NO_SVN_DEPENDS' in flags and strtobool(flags['NO_SVN_DEPENDS']) else False + doc = _Markdown(arc_root, dump_all_conf_docs, not no_svn) + for efile in jdata: + for entry in jdata[efile]: + doc.process_entry(efile, entry) + res.stdout += doc.dump() + + return res diff --git a/devtools/ya/handlers/dump/ya.make b/devtools/ya/handlers/dump/ya.make new file mode 100644 index 0000000000..adbc66ad9f --- /dev/null +++ b/devtools/ya/handlers/dump/ya.make @@ -0,0 +1,52 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.dump + __init__.py + gen_conf_docs.py +) + +PEERDIR( + contrib/python/pathlib2 + contrib/python/six + contrib/python/toml + devtools/ya/app + devtools/ya/build + devtools/ya/build/build_facade + devtools/ya/build/build_opts + devtools/ya/build/genconf + devtools/ya/build/ymake2 + devtools/ya/core + devtools/ya/core/config + devtools/ya/core/imprint + devtools/ya/core/yarg + devtools/ya/exts + devtools/ya/test/dartfile + devtools/ya/test/explore + devtools/ya/yalibrary/debug_store + devtools/ya/yalibrary/debug_store/processor + devtools/ya/yalibrary/debug_store/store + devtools/ya/yalibrary/tools + devtools/ya/yalibrary/vcs + devtools/ya/yalibrary/vcs/vcsversion + devtools/ya/yalibrary/yandex/sandbox/misc + library/python/fs + library/python/func + library/python/tmp +) + +IF (NOT YA_OPENSOURCE) + PEERDIR( + devtools/ya/handlers/dump/arcadia_specific + devtools/ya/handlers/dump/debug + ) +ENDIF() + +END() + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/gc/__init__.py b/devtools/ya/handlers/gc/__init__.py new file mode 100644 index 0000000000..824a35edf2 --- /dev/null +++ b/devtools/ya/handlers/gc/__init__.py @@ -0,0 +1,342 @@ +from __future__ import absolute_import +import glob +import os +import logging +import sys +import time +import six + +import app +import core.yarg +import core.common_opts +import core.config as cc +import build.ya_make as ym +from exts import fs +from core.common_opts import CustomBuildRootOptions +from build.build_opts import LocalCacheOptions, DistCacheSetupOptions, parse_size_arg, parse_timespan_arg +from exts.windows import on_win +from yalibrary.runner import result_store +import yalibrary.toolscache as tc + +if six.PY3: + long = int + +logger = logging.getLogger(__name__) + + +def _to_size_in_gb(size): + try: + return int(float(size) * 1024 * 1024 * 1024) + except ValueError: + pass + + return parse_size_arg(size) + + +def _to_size_in_mb(size): + try: + return int(float(size) * 1024 * 1024) + except ValueError: + pass + + return parse_size_arg(size) + + +def _to_timespan_in_hours(timespan): + try: + return int(float(timespan) * 60 * 60) + except ValueError: + pass + + return parse_timespan_arg(timespan) + + +class CollectCacheOptions(LocalCacheOptions): + def __init__(self): + super(CollectCacheOptions, self).__init__() + self.object_size_limit = None + self.age_limit = None + self.symlinks_ttl = 0 + + def consumer(self): + return super(CollectCacheOptions, self).consumer() + [ + core.yarg.ArgConsumer( + ['--size-limit'], + help='Strip build cache to size (in GiB if not set explicitly)', + hook=core.yarg.SetValueHook( + 'cache_size', + transform=_to_size_in_gb, + default_value=lambda x: str(_to_size_in_gb(x) * 1.0 / 1024 / 1024 / 1024), + ), + group=core.yarg.BULLET_PROOF_OPT_GROUP, + visible=False, + ), + core.yarg.ArgConsumer( + ['--object-size-limit'], + help='Strip build cache from large objects (in MiB if not set explicitly)', + hook=core.yarg.SetValueHook('object_size_limit', transform=_to_size_in_mb), + group=core.yarg.BULLET_PROOF_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--age-limit'], + help='Strip build cache from old objects (in hours if not set explicitly)', + hook=core.yarg.SetValueHook('age_limit', transform=_to_timespan_in_hours), + group=core.yarg.BULLET_PROOF_OPT_GROUP, + ), + ] + + +class GarbageCollectionYaHandler(core.yarg.CompositeHandler): + def __init__(self): + core.yarg.CompositeHandler.__init__(self, description='Collect garbage') + self['cache'] = core.yarg.OptsHandler( + action=app.execute(action=do_cache, respawn=app.RespawnType.OPTIONAL), + description='Strip build cache and old build directories', + opts=[core.common_opts.ShowHelpOptions(), CollectCacheOptions(), CustomBuildRootOptions()], + visible=True, + ) + self['dist_cache'] = core.yarg.OptsHandler( + action=app.execute(action=do_strip_yt_cache, respawn=app.RespawnType.NONE), + description='Strip distributed (YT) cache', + opts=[core.common_opts.ShowHelpOptions(), DistCacheSetupOptions()], + visible=False, + ) + + +class FilterBySize(object): + def __init__(self, size_limit): + self.size_limit = size_limit + self.total_size = 0 + self._items = {} + + def __call__(self, item): + if item.uid in self._items: + return False + size = item.size + self.total_size += size + self._items[item.uid] = item + return self.total_size < self.size_limit + + +class FilterByObjectSize(object): + def __init__(self, size_limit): + self.size_limit = size_limit + + def __call__(self, item): + return item.size < self.size_limit + + +class FilterByAge(object): + def __init__(self, age_limit): + self.age_limit = age_limit + self.now = time.time() + + def __call__(self, item): + return item.timestamp > self.now - self.age_limit + + +def _clean_dir_selectively(logs, file_dir, file_name): + try: + for i in os.listdir(logs): + date_dir = os.path.join(logs, i) + if date_dir != file_dir: + fs.remove_tree_safe(date_dir) + + if file_dir: + for i in os.listdir(file_dir): + if i != file_name: + fs.remove_tree_safe(os.path.join(file_dir, i)) + except OSError as e: + logger.debug('Cleaning %s root failed %s', logs, e) + + +def _clean_logs(): + logger.debug('Cleaning logs root') + + try: + import app_ctx + + file_dir, file_name = os.path.split(app_ctx.file_log) + except Exception as e: + file_dir, file_name = ('', '') + logger.debug('Log name was not obtained %s', e) + + logs = os.path.join(cc.misc_root(), 'logs') + _clean_dir_selectively(logs, file_dir, file_name) + + +def _clean_evlogs(): + logger.debug('Cleaning evlogs root') + + try: + import app_ctx + + file_dir, file_name = os.path.split(app_ctx.evlog._fileobj.name) + except Exception as e: + file_dir, file_name = ('', '') + logger.debug('Log name was not obtained %s', e) + + evlogs = os.path.join(cc.misc_root(), 'evlogs') + _clean_dir_selectively(evlogs, file_dir, file_name) + + +def _clean_tools(): + running_ya_bin_dir = os.path.dirname(sys.argv[0]) + base_running_ya_bin_dir = os.path.basename(running_ya_bin_dir).replace('_d', '') + errors = 0 + for i in os.listdir(os.path.dirname(cc.tool_root())): + full_path = os.path.join(os.path.dirname(cc.tool_root()), i) + if i == 'v4': + if len(glob.glob(os.path.join(full_path, '*.corrupted'))) == 0: + tc.unlock_all() + continue + else: + errors += 1 + elif full_path == running_ya_bin_dir: + continue + elif os.path.isfile(full_path): + try: + if os.path.getsize(full_path) < 1024 * 100: + with open(full_path, 'r') as f: + if base_running_ya_bin_dir in f.read(): + continue + except Exception: + errors += 1 + logger.debug('Cleaning tool %s', full_path) + fs.remove_tree_safe(full_path) + return errors + + +def do_cache(opts): + build_root = opts.custom_build_directory or cc.build_root() + + cache = None + try: + cache = ym.make_cache(opts, build_root) + except Exception: + logger.exception("While initializing cache") + + lock = ym.make_lock(opts, build_root, write_lock=True) + + with lock: + if opts.cache_stat: + import app_ctx + + if not cache: + app_ctx.display.emit_message("Cache not initialized, can't show stats") + else: + cache.analyze(app_ctx.display) + errors = 0 + else: + errors = _do_collect_cache(cache, build_root, opts) + + if cache: + cache.flush() + + logger.debug( + "ya gc stats %s", + { + 'cache_size': opts.cache_size, + 'object_size_limit': opts.object_size_limit, + 'age_limit': opts.age_limit, + 'symlinks_ttl': opts.symlinks_ttl, + 'errors': errors, + }, + ) + + +def _do_collect_cache(cache, build_root, opts): + logger.debug('Cleaning tmp root') + fs.remove_tree_safe(cc.tmp_path()) + + logger.debug('Cleaning snowden root') + fs.remove_tree_safe(os.path.join(cc.misc_root(), 'snowden')) + + logger.debug('Cleaning build root') + fs.remove_tree_safe(os.path.join(build_root, 'build_root')) + + logger.debug('Cleaning conf root') + fs.remove_tree_safe(os.path.join(build_root, 'conf')) + + errors = _clean_tools() + + logger.debug('Cleaning tmp root') + fs.remove_tree_safe(os.path.join(cc.misc_root(), 'tmp')) + + _clean_logs() + _clean_evlogs() + + if not on_win(): + logger.debug('Cleaning symres') + symres_dir = os.path.join(build_root, 'symres') + src_dir = cc.find_root(fail_on_error=False) or "" + if os.path.isdir(symres_dir): + logger.debug('Cleaning symres %s for %s, ttl=%s', symres_dir, src_dir, opts.symlinks_ttl) + result_store.SymlinkResultStore(symres_dir, src_dir).sieve( + state=None, ttl=opts.symlinks_ttl, cleanup=opts.symlinks_ttl == 0 + ) + + if hasattr(cache, '_store_path'): + for dir in os.listdir(os.path.join(build_root, 'cache')): + full_path = os.path.join(build_root, 'cache', dir) + if cache._store_path == full_path: + if len(glob.glob(os.path.join(full_path, '*.corrupted'))) == 0: + continue + else: + errors += 1 + logger.debug('Cleaning cache directory %s', full_path) + fs.remove_tree_safe(full_path) + + if opts.cache_size is not None and opts.object_size_limit is None and opts.age_limit is None: + logger.debug('Cleaning for total size %s', opts.cache_size) + if hasattr(cache, 'strip'): + cache.strip(FilterBySize(opts.cache_size)) + elif hasattr(cache, 'strip_total_size'): + cache.strip_total_size(opts.cache_size) + + from yalibrary.toolscache import tc_force_gc + + tc_force_gc(opts.cache_size) + + if opts.object_size_limit is not None: + logger.debug('Cleaning for object size %s', opts.object_size_limit) + if hasattr(cache, 'strip'): + cache.strip(FilterByObjectSize(opts.object_size_limit)) + elif hasattr(cache, 'strip_max_object_size'): + cache.strip_max_object_size(opts.object_size_limit) + + if opts.age_limit is not None: + logger.debug('Cleaning for age %s', opts.age_limit) + if hasattr(cache, 'strip'): + cache.strip(FilterByAge(opts.age_limit)) + elif hasattr(cache, 'strip_max_age'): + cache.strip_max_age(opts.age_limit) + + return errors + + +def do_strip_yt_cache(opts): + try: + from yalibrary.store.yt_store import yt_store + except ImportError as e: + logger.warn("YT store is not available: %s", e) + + token = opts.yt_token or opts.oauth_token + cache = yt_store.YtStore( + opts.yt_proxy, + opts.yt_dir, + None, + token=token, + readonly=opts.yt_readonly, + max_cache_size=opts.yt_max_cache_size, + ttl=opts.yt_store_ttl, + ) + + counters = cache.strip() + if counters: + logger.info( + 'Deleted: meta rows:%d, data rows:%d, net data size:%d', + counters['meta_rows'], + counters['data_rows'], + counters['data_size'], + ) diff --git a/devtools/ya/handlers/gc/ya.make b/devtools/ya/handlers/gc/ya.make new file mode 100644 index 0000000000..1b8296d1ee --- /dev/null +++ b/devtools/ya/handlers/gc/ya.make @@ -0,0 +1,37 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.gc + __init__.py +) + +PEERDIR( + contrib/python/six + devtools/ya/app + devtools/ya/build + devtools/ya/build/build_opts + devtools/ya/core + devtools/ya/core/config + devtools/ya/core/yarg + devtools/ya/exts + devtools/ya/yalibrary/runner + devtools/ya/yalibrary/store + devtools/ya/yalibrary/toolscache + library/python/fs + library/python/windows +) + +IF (NOT YA_OPENSOURCE) + PEERDIR( + devtools/ya/yalibrary/store/yt_store # see YA-938 + ) +ENDIF() + +END() + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/gen_config/__init__.py b/devtools/ya/handlers/gen_config/__init__.py new file mode 100644 index 0000000000..e771dac871 --- /dev/null +++ b/devtools/ya/handlers/gen_config/__init__.py @@ -0,0 +1,48 @@ +from __future__ import absolute_import +import core.common_opts +import core.yarg + +from . import gen_config + + +class GenConfigOptions(core.yarg.Options): + def __init__(self): + self.output = None + self.dump_defaults = False + + @staticmethod + def consumer(): + return [ + core.yarg.SingleFreeArgConsumer( + help='ya.conf', + hook=core.yarg.SetValueHook('output'), + required=False, + ), + core.yarg.ArgConsumer( + ['--dump-defaults'], + help='Dump default values as JSON', + hook=core.yarg.SetConstValueHook('dump_defaults', True), + ), + ] + + +class GenConfigYaHandler(core.yarg.OptsHandler): + description = 'Generate default ya config' + + def __init__(self): + self._root_handler = None + super(GenConfigYaHandler, self).__init__( + action=self.do_generate, + description=self.description, + opts=[ + core.common_opts.ShowHelpOptions(), + GenConfigOptions(), + ], + ) + + def handle(self, root_handler, args, prefix): + self._root_handler = root_handler + super(GenConfigYaHandler, self).handle(root_handler, args, prefix) + + def do_generate(self, args): + gen_config.generate_config(self._root_handler, args.output, args.dump_defaults) diff --git a/devtools/ya/handlers/gen_config/gen_config.py b/devtools/ya/handlers/gen_config/gen_config.py new file mode 100644 index 0000000000..56a0d6fc64 --- /dev/null +++ b/devtools/ya/handlers/gen_config/gen_config.py @@ -0,0 +1,282 @@ +from __future__ import absolute_import +import sys +import itertools +import collections +import json + +import toml + +import core.yarg +import six + +toml_encoder = toml.TomlEncoder() + +NOT_SET = -1 + + +def is_jsonable(x): + try: + json.dumps(x) + return True + except (TypeError, OverflowError): + return False + + +def iter_ya_options(root_handler, all_opts=False): + # prefer setvalue over setconstvalue to provide correct description and default value + hook_prefer_order = [core.yarg.SetValueHook, core.yarg.SetConstValueHook] + + def get_hook_priority(hook, consumer): + if isinstance(consumer, core.yarg.ConfigConsumer): + return -1 + + htype = type(hook) + if htype in hook_prefer_order: + pos = hook_prefer_order.index(htype) + else: + pos = len(hook_prefer_order) + return len(hook_prefer_order) - pos + + for handler_name, command in six.iteritems(root_handler.sub_handlers): + if not command.visible: + continue + + top_handler = command.command() + handlers = collections.deque([([handler_name], top_handler)]) + + while handlers: + trace, handler = handlers.pop() + + sub_handlers = handler.sub_handlers or {} + for sub_name in sorted(sub_handlers): + handlers.append((trace + [sub_name], sub_handlers[sub_name])) + + options = getattr(handler, "options", None) + if not options: + continue + + if all_opts: + for opt in options: + for name in vars(opt).keys(): + default = getattr(opt, name) + if is_jsonable(default): + yield name, "", default, "", "", "" + continue + + top_consumer = options.consumer() + if isinstance(top_consumer, core.yarg.Compound): + consumers = top_consumer.parts + else: + consumers = [top_consumer] + + # pairwise config option with command line name if it's possible + params = {} + for consumer in consumers: + if not isinstance(consumer, (core.yarg.ArgConsumer, core.yarg.ConfigConsumer)): + continue + + hook = consumer.hook + if not hook: + continue + + opt_name = getattr(hook, 'name', None) + if not opt_name or opt_name.startswith("_"): + continue + + if opt_name not in params: + params[opt_name] = { + 'configurable': False, + } + + entry = params[opt_name] + priority = get_hook_priority(hook, consumer) + + entry['configurable'] |= isinstance(consumer, core.yarg.ConfigConsumer) + if priority > entry.get('_priority', -2): + entry['cmdline_names'] = get_arg_consumer_names(consumer) or entry.get('cmdline_names') + entry['trace'] = trace or entry.get('trace') + entry['description'] = getattr(consumer, "help", "") or entry.get('description') + entry['group'] = getattr(consumer, "group", "") or entry.get('group', 0) + # default value might be empty list or dict + if getattr(options, opt_name, None) is None: + entry['default'] = entry.get('default') + else: + entry['default'] = getattr(options, opt_name) + entry['_priority'] = priority + + for name, x in params.items(): + if x['configurable']: + yield name, x['cmdline_names'], x['default'], x['group'], x['description'], x['trace'] + + +def get_arg_consumer_names(consumer): + if isinstance(consumer, core.yarg.ArgConsumer): + return list([_f for _f in [consumer.short_name, consumer.long_name] if _f]) + return [] + + +def split(iterable, func): + d1 = [] + d2 = [] + for entry in iterable: + if func(entry): + d1.append(entry) + else: + d2.append(entry) + return d1, d2 + + +def get_comment(entry): + parts = [] + if entry['desc']: + parts.append(entry['desc']) + if entry['cmdline_names']: + parts.append("({})".format(", ".join(entry['cmdline_names']))) + return " ".join(parts) + + +def compress_keys(data, result, trace=None): + trace = trace or [] + for key, val in six.iteritems(data): + if isinstance(val, dict): + result[".".join(trace + [key])] = val + compress_keys(val, result, trace + [key]) + + +def dump_options(entries, output, config, section="", print_comment=True): + if section: + output.append("[{}]".format(section)) + for entry in sorted(entries, key=lambda x: x['name']): + parts = [] + if print_comment: + comment = get_comment(entry) + if comment: + parts.append("# {}".format(comment)) + name = entry['name'] + if name in config and config[name] != entry['default']: + parts.append("{} = {}".format(name, toml_encoder.dump_value(config[name]))) + else: + parts.append("# {} = {}".format(name, toml_encoder.dump_value(entry['default']))) + + output.append("\n".join(parts)) + + +def dump_subgroups(subgroups, output, config, section="", print_comment=True): + def get_section(name): + return ".".join([_f for _f in [section, name] if _f]) + + for entry in subgroups: + parts = [] + if print_comment: + comment = get_comment(entry) + if comment: + parts.append("# {}".format(comment)) + + name = entry['name'] + if name in config and config[name] != entry['default']: + parts.append("[{}]".format(get_section(name))) + for key, val in config[name].items(): + parts.append("{} = {}".format(key, toml_encoder.dump_value(val))) + else: + parts.append("# [{}]".format(get_section(name))) + for key, val in entry['default'].items(): + parts.append("# {} = {}".format(key, toml_encoder.dump_value(val))) + + output.append("\n".join(parts)) + + +def dump_config(options, handler_map, user_config): + def sort_func(x): + return getattr(x['group'], 'index', NOT_SET) + + def get_group_title(name): + return "{} {} {}".format("=" * 10, name, "=" * (65 - len(name))) + + blocks = [ + "# Save config to the junk/{USER}/ya.conf or to the ~/.ya/ya.conf\n# For more info see https://docs.yandex-team.ru/yatool/commands/gen_config" + ] + + # dump all options + subgroups = [] + data = sorted(options.values(), key=sort_func) + for group_index, entries in itertools.groupby(data, sort_func): + entries = list(entries) + + if group_index != NOT_SET: + group_name = entries[0]['group'].name + if group_name: + blocks.append("# " + get_group_title(group_name)) + + subs, opts = split(entries, lambda x: isinstance(x['default'], dict)) + dump_options(opts, blocks, user_config) + # move all subgroup options out of the default section + # otherwise every option defined after section will be related to the section + subgroups += subs + + if subgroups: + blocks.append("# " + get_group_title("Various table options")) + blocks.append("# Uncomment table name with parameters") + dump_subgroups(subgroups, blocks, user_config) + + # Save user redefined opts for specific handlers + specific_opts = {} + user_handler_map = {} + compress_keys(user_config, user_handler_map) + for section, keys in six.iteritems(user_handler_map): + if section not in handler_map: + continue + entries = [] + for optname in keys: + if optname in handler_map[section]: + entries.append(options[optname]) + if entries: + specific_opts[section] = entries + + if specific_opts: + blocks.append("# " + get_group_title("Redefined options for specific handlers")) + for section, entries in sorted(specific_opts.items(), key=lambda x: x[0]): + subs, opts = split(entries, lambda x: isinstance(x['default'], dict)) + if opts: + dump_options(opts, blocks, user_handler_map[section], section, print_comment=False) + if subgroups: + dump_subgroups(subs, blocks, user_handler_map[section], section, print_comment=False) + + return "\n#\n".join(blocks) + + +def generate_config(root_handler, output=None, dump_defaults=None): + # Don't load global config files to avoid penetration of the global options into user config + config_files = core.yarg.get_config_files(global_config=False) + user_config = core.yarg.load_config(config_files) + + options = {} + handler_map = {} + for name, cmdline_names, default, group, desc, trace in iter_ya_options(root_handler, dump_defaults): + if name not in options: + options[name] = { + 'cmdline_names': cmdline_names, + 'default': default, + 'desc': desc, + 'group': group, + 'name': name, + } + else: + entry = options[name] + entry['desc'] = entry['desc'] or desc + entry['group'] = entry['group'] or group + + target = ".".join(trace) + if target not in handler_map: + handler_map[target] = [] + handler_map[target].append(name) + + if dump_defaults: + json.dump({k: v['default'] for k, v in options.items()}, sys.stdout, indent=2) + return + + data = dump_config(options, handler_map, user_config) + if output: + with open(output, 'w') as afile: + afile.write(data) + else: + sys.stdout.write(data + '\n') diff --git a/devtools/ya/handlers/gen_config/ya.make b/devtools/ya/handlers/gen_config/ya.make new file mode 100644 index 0000000000..b5fdeece7c --- /dev/null +++ b/devtools/ya/handlers/gen_config/ya.make @@ -0,0 +1,23 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.gen_config + __init__.py + gen_config.py +) + +PEERDIR( + contrib/python/six + contrib/python/toml + devtools/ya/core + devtools/ya/core/yarg +) + +END() + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/ide/__init__.py b/devtools/ya/handlers/ide/__init__.py new file mode 100644 index 0000000000..4d28d29a69 --- /dev/null +++ b/devtools/ya/handlers/ide/__init__.py @@ -0,0 +1,734 @@ +from __future__ import absolute_import +import os.path + +import six + +import core.yarg +import core.config +import core.common_opts + +import build.build_opts +import build.compilation_database as bcd + +import ide.ide_common +import ide.msvs +import ide.msvs_lite +import ide.clion2016 +import ide.idea +import ide.qt +import ide.remote_ide_qt +import ide.goland +import ide.pycharm +import ide.venv +import ide.vscode_all +import ide.vscode_clangd +import ide.vscode_go +import ide.vscode_py +import ide.vscode_ts +import ide.vscode.opts + +import yalibrary.platform_matcher as pm + +import app +import app_config + +if app_config.in_house: + import devtools.ya.ide.fsnotifier + +from core.yarg.help_level import HelpLevel + +if six.PY3: + import ide.gradle + + +class TidyOptions(core.yarg.Options): + def __init__(self): + self.setup_tidy = False + + @staticmethod + def consumer(): + return [ + core.yarg.ArgConsumer( + ['--setup-tidy'], + help="Setup default arcadia's clang-tidy config in a project", + hook=core.yarg.SetConstValueHook('setup_tidy', True), + group=core.yarg.BULLET_PROOF_OPT_GROUP, + ), + core.yarg.ConfigConsumer('setup_tidy'), + ] + + +class CLionOptions(core.yarg.Options): + CLION_OPT_GROUP = core.yarg.Group('CLion project options', 0) + + def __init__(self): + self.filters = [] + self.lite_mode = False + self.remote_toolchain = None + self.remote_deploy_config = None + self.remote_repo_path = None + self.remote_build_path = None + self.remote_deploy_host = None + self.use_sync_server = False + self.content_root = None + self.strip_non_final_targets = False + self.full_targets = False + self.add_py_targets = False + + @staticmethod + def consumer(): + return [ + core.yarg.ArgConsumer( + ['--filter', '-f'], + help='Only consider filtered content', + hook=core.yarg.SetAppendHook('filters'), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--mini', '-m'], + help='Lite mode for solution (fast open, without build)', + hook=core.yarg.SetConstValueHook('lite_mode', True), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--remote-toolchain'], + help='Generate configurations for remote toolchain with this name', + hook=core.yarg.SetValueHook('remote_toolchain'), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--remote-deploy-config'], + help='Name of the remote server configuration tied to the remote toolchain', + hook=core.yarg.SetValueHook('remote_deploy_config'), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--remote-repo-path'], + help='Path to the arc repository at the remote host', + hook=core.yarg.SetValueHook('remote_repo_path'), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--remote-build-path'], + help='Path to the directory for CMake output at the remote host', + hook=core.yarg.SetValueHook('remote_build_path'), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--remote-host'], + help='Hostname associated with remote server configuration', + hook=core.yarg.SetValueHook('remote_deploy_host'), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--use-sync-server'], + help='Deploy local files via sync server instead of file watchers', + hook=core.yarg.SetConstValueHook('use_sync_server', True), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--project-root', '-r'], + help='Root directory for a CLion project', + hook=core.yarg.SetValueHook('content_root'), + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--strip-non-final-targets'], + hook=core.yarg.SetConstValueHook('strip_non_final_targets', True), + help='Do not create target for non-final nodes', + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--full-targets'], + hook=core.yarg.SetConstValueHook('full_targets', True), + help='Old Mode: Enable full targets graph generation for project.', + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--add-py3-targets'], + hook=core.yarg.SetConstValueHook('add_py_targets', True), + help='Add Python 3 targets to project', + group=CLionOptions.CLION_OPT_GROUP, + ), + core.yarg.ConfigConsumer('filters'), + core.yarg.ConfigConsumer('lite_mode'), + core.yarg.ConfigConsumer('remote_toolchain'), + core.yarg.ConfigConsumer('remote_deploy_config'), + core.yarg.ConfigConsumer('remote_repo_path'), + core.yarg.ConfigConsumer('remote_build_path'), + core.yarg.ConfigConsumer('remote_deploy_host'), + core.yarg.ConfigConsumer('use_sync_server'), + core.yarg.ConfigConsumer('content_root'), + core.yarg.ConfigConsumer('strip_non_executable_target'), + core.yarg.ConfigConsumer('full_targets'), + core.yarg.ConfigConsumer('add_py_targets'), + ] + + def postprocess2(self, params): + if ' ' in params.project_title: + raise core.yarg.ArgsValidatingException('Clion project title should not contain space symbol') + if params.add_py_targets and params.full_targets: + raise core.yarg.ArgsValidatingException('--add-py-targets must be used without --full-targets') + + +class IdeaOptions(core.yarg.Options): + IDEA_OPT_GROUP = core.yarg.Group('Idea project options', 0) + IDE_PLUGIN_INTEGRATION_GROUP = core.yarg.Group('Integration wuth IDE plugin', 1) + + def __init__(self): + self.idea_project_root = None + self.local = False + self.group_modules = None + self.dry_run = False + self.ymake_bin = None + self.iml_in_project_root = False + self.iml_keep_relative_paths = False + self.idea_files_root = None + self.project_name = None + self.minimal = False + self.directory_based = True + self.omit_test_data = False + self.with_content_root_modules = False + self.external_content_root_modules = [] + self.generate_tests_run = False + self.generate_tests_for_deps = False + self.separate_tests_modules = False + self.auto_exclude_symlinks = False + self.exclude_dirs = [] + self.with_common_jvm_args_in_junit_template = False + self.with_long_library_names = False + self.copy_shared_index_config = False + self.idea_jdk_version = None + self.regenerate_with_project_update = False + self.project_update_targets = [] + self.project_update_kind = None + + @staticmethod + def consumer(): + return [ + core.yarg.ArgConsumer( + ['-r', '--project-root'], + help='IntelliJ IDEA project root path', + hook=core.yarg.SetValueHook('idea_project_root'), + group=IdeaOptions.IDEA_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['-P', '--project-output'], + help='IntelliJ IDEA project root path. Please, use instead of -r', + hook=core.yarg.SetValueHook('idea_project_root'), + group=IdeaOptions.IDEA_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['-l', '--local'], + help='Only recurse reachable projects are idea modules', + hook=core.yarg.SetConstValueHook('local', True), + group=IdeaOptions.IDEA_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--group-modules'], + help='Group idea modules according to paths: (tree, flat)', + hook=core.yarg.SetValueHook('group_modules', values=('tree', 'flat')), + group=IdeaOptions.IDEA_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['-n', '--dry-run'], + help='Emulate create project, but do nothing', + hook=core.yarg.SetConstValueHook('dry_run', True), + group=IdeaOptions.IDEA_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--ymake-bin'], help='Path to ymake binary', hook=core.yarg.SetValueHook('ymake_bin'), visible=False + ), + core.yarg.ArgConsumer( + ['--iml-in-project-root'], + help='Store ".iml" files in project root tree(stores in source root tree by default)', + hook=core.yarg.SetConstValueHook('iml_in_project_root', True), + ), + core.yarg.ArgConsumer( + ['--iml-keep-relative-paths'], + help='Keep relative paths in ".iml" files (works with --iml-in-project-root)', + hook=core.yarg.SetConstValueHook('iml_keep_relative_paths', True), + ), + core.yarg.ArgConsumer( + ['--idea-files-root'], + help='Root for .ipr and .iws files', + hook=core.yarg.SetValueHook('idea_files_root'), + ), + core.yarg.ArgConsumer( + ['--project-name'], + help='Idea project name (.ipr and .iws file)', + hook=core.yarg.SetValueHook('project_name'), + ), + core.yarg.ArgConsumer( + ['--ascetic'], + help='Create the minimum set of project settings', + hook=core.yarg.SetConstValueHook('minimal', True), + ), + core.yarg.ArgConsumer( + ['--directory-based'], + help='Create project in actual (directory based) format', + hook=core.yarg.SetConstValueHook('directory_based', True), + visible=False, + ), + core.yarg.ArgConsumer( + ['--omit-test-data'], + help='Do not export test_data', + hook=core.yarg.SetConstValueHook('omit_test_data', True), + ), + core.yarg.ArgConsumer( + ['--with-content-root-modules'], + help='Generate content root modules', + hook=core.yarg.SetConstValueHook('with_content_root_modules', True), + ), + core.yarg.ArgConsumer( + ['--external-content-root-module'], + help='Add external content root modules', + hook=core.yarg.SetAppendHook('external_content_root_modules'), + ), + core.yarg.ArgConsumer( + ['--generate-junit-run-configurations'], + help='Generate run configuration for junit tests', + hook=core.yarg.SetConstValueHook('generate_tests_run', True), + ), + core.yarg.ArgConsumer( + ['--generate-tests-for-dependencies'], + help='Generate tests for PEERDIR dependencies', + hook=core.yarg.SetConstValueHook('generate_tests_for_deps', True), + ), + core.yarg.ArgConsumer( + ['--separate-tests-modules'], + help='Do not merge tests modules with their own libraries', + hook=core.yarg.SetConstValueHook('separate_tests_modules', True), + ), + core.yarg.ArgConsumer( + ['--auto-exclude-symlinks'], + help='Add all symlink-dirs in modules to exclude dirs', + hook=core.yarg.SetConstValueHook('auto_exclude_symlinks', True), + ), + core.yarg.ArgConsumer( + ['--exclude-dirs'], + help='Exclude dirs with specific names from all modules', + hook=core.yarg.SetAppendHook('exclude_dirs'), + ), + core.yarg.ArgConsumer( + ['--with-common-jvm-args-in-junit-template'], + help='Add common JVM_ARGS flags to default junit template', + hook=core.yarg.SetConstValueHook('with_common_jvm_args_in_junit_template', True), + ), + core.yarg.ArgConsumer( + ['--with-long-library-names'], + help='Generate long library names', + hook=core.yarg.SetConstValueHook('with_long_library_names', True), + ), + core.yarg.ArgConsumer( + ['--copy-shared-index-config'], + help='Copy project config for Shared Indexes if exist', + hook=core.yarg.SetConstValueHook('copy_shared_index_config', True), + ), + core.yarg.ArgConsumer( + ['--idea-jdk-version'], + help='Project JDK version', + hook=core.yarg.SetValueHook('idea_jdk_version'), + ), + core.yarg.ArgConsumer( + ['-U', '--regenerate-with-project-update'], + help='Run `ya project update` upon regeneration from Idea', + group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP, + hook=core.yarg.SetConstValueHook('regenerate_with_project_update', True), + ), + core.yarg.ArgConsumer( + ['--project-update-targets'], + help='Run `ya project update` for this dirs upon regeneration from Idea', + hook=core.yarg.SetAppendHook('project_update_targets'), + group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP, + visible=HelpLevel.ADVANCED, + ), + core.yarg.ArgConsumer( + ['--project-update-kind'], + help='Type of a project to use in `ya project update` upon regernation from Idea', + hook=core.yarg.SetValueHook('project_update_kind'), + group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP, + visible=HelpLevel.ADVANCED, + ), + core.yarg.ConfigConsumer('idea_project_root'), + core.yarg.ConfigConsumer('local'), + core.yarg.ConfigConsumer('group_modules'), + core.yarg.ConfigConsumer('dry_run'), + core.yarg.ConfigConsumer('iml_in_project_root'), + core.yarg.ConfigConsumer('iml_keep_relative_paths'), + core.yarg.ConfigConsumer('idea_files_root'), + core.yarg.ConfigConsumer('project_name'), + core.yarg.ConfigConsumer('minimal'), + core.yarg.ConfigConsumer('directory_based'), + core.yarg.ConfigConsumer('omit_test_data'), + core.yarg.ConfigConsumer('with_content_root_modules'), + core.yarg.ConfigConsumer('external_content_root_modules'), + core.yarg.ConfigConsumer('generate_tests_run'), + core.yarg.ConfigConsumer('generate_tests_for_deps'), + core.yarg.ConfigConsumer('separate_tests_modules'), + core.yarg.ConfigConsumer('auto_exclude_symlinks'), + core.yarg.ConfigConsumer('exclude_dirs'), + core.yarg.ConfigConsumer('with_common_jvm_args_in_junit_template'), + core.yarg.ConfigConsumer('with_long_library_names'), + core.yarg.ConfigConsumer('copy_shared_index_config'), + core.yarg.ConfigConsumer('idea_jdk_version'), + core.yarg.ConfigConsumer('regenarate_with_project_update'), + core.yarg.ConfigConsumer('project_update_targets'), + core.yarg.ConfigConsumer('project_update_kind'), + ] + + def postprocess(self): + if self.idea_project_root is None: + raise core.yarg.ArgsValidatingException('Idea project root(-r, --project-root) must be specified.') + + self.idea_project_root = os.path.abspath(self.idea_project_root) + + if self.iml_keep_relative_paths and not self.iml_in_project_root: + raise core.yarg.ArgsValidatingException( + '--iml-keep-relative-paths can be used only with --iml-in-project-root' + ) + + if self.generate_tests_run and not self.directory_based: + raise core.yarg.ArgsValidatingException( + 'run configurations may be generated only for directory-based project' + ) + + for p in self.exclude_dirs: + if os.path.isabs(p): + raise core.yarg.ArgsValidatingException('Absolute paths are not allowed in --exclude-dirs') + + +class GradleOptions(core.yarg.Options): + GRADLE_OPT_GROUP = core.yarg.Group('Gradle project options', 0) + + def __init__(self): + self.gradle_name = None + + @staticmethod + def consumer(): + return [ + core.yarg.ArgConsumer( + ['--gradle-name'], + help='Set project name manually', + hook=core.yarg.SetValueHook('gradle_name'), + group=GradleOptions.GRADLE_OPT_GROUP, + ) + ] + + +class PycharmOptions(core.yarg.Options): + PYCHARM_OPT_GROUP = core.yarg.Group('Pycharm project options', 0) + PYTHON_WRAPPER_NAME = 'pycharm_python_wrapper' + + def __init__(self): + self.only_generate_wrapper = False + self.wrapper_name = PycharmOptions.PYTHON_WRAPPER_NAME + self.list_ide = False + self.ide_version = None + + @staticmethod + def consumer(): + return [ + core.yarg.ArgConsumer( + ['--only-generate-wrapper'], + help="Don not generate Pycharm project, only wrappers", + hook=core.yarg.SetConstValueHook('only_generate_wrapper', True), + group=PycharmOptions.PYCHARM_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--wrapper-name'], + help='Name of generated python wrapper. Use `python` for manual adding wrapper as Python SDK.', + hook=core.yarg.SetValueHook('wrapper_name'), + group=PycharmOptions.PYCHARM_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--list-ide'], + help='List available JB IDE for patching SDK list.', + hook=core.yarg.SetConstValueHook('list_ide', True), + group=PycharmOptions.PYCHARM_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--ide-version'], + help='Change IDE version for patching SDK list. Available IDE: {}'.format( + ", ".join(ide.pycharm.find_available_ide()) + ), + hook=core.yarg.SetValueHook('ide_version'), + group=PycharmOptions.PYCHARM_OPT_GROUP, + ), + ] + + def postprocess(self): + if PycharmOptions.PYTHON_WRAPPER_NAME != self.wrapper_name and not self.only_generate_wrapper: + raise core.yarg.ArgsValidatingException( + "Custom wrapper name can be used with option --only-generate-wrapper" + ) + + +MSVS_OPTS = ide.msvs.MSVS_OPTS + [ide.ide_common.YaExtraArgsOptions(), core.common_opts.YaBin3Options()] + + +def gen_msvs_solution(params): + impl = ide.msvs_lite if params.lite else ide.msvs + return impl.gen_msvs_solution(params) + + +def get_description(text, ref_name): + if app_config.in_house: + ref = { + "c": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#c", + "golang": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#golang", + "multi": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#multi", + "python": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#python", + "typescript": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#typescript", + }[ref_name] + return "{}\nDocs: [[c:dark-cyan]]{}[[rst]]".format(text, ref) + else: + return text + + +class IdeYaHandler(core.yarg.CompositeHandler): + description = 'Generate project for IDE' + + def __init__(self): + core.yarg.CompositeHandler.__init__(self, description=self.description) + self['msvs'] = core.yarg.OptsHandler( + action=app.execute(gen_msvs_solution), + description='[[imp]]ya ide msvs[[rst]] is deprecated, please use clangd-based tooling instead', + opts=MSVS_OPTS, + examples=[ + core.yarg.UsageExample( + '{prefix} util/generic util/datetime', + 'Generate solution for util/generic, util/datetime and all their dependencies', + ), + core.yarg.UsageExample('{prefix} -P Output', 'Generate solution in Output directory'), + core.yarg.UsageExample('{prefix} -T my_solution', 'Generate solution titled my_solution.sln'), + ], + visible=(pm.my_platform() == 'win32'), + ) + self['clion'] = core.yarg.OptsHandler( + action=app.execute(ide.clion2016.do_clion), + description='[[imp]]ya ide clion[[rst]] is deprecated, please use clangd-based tooling instead', + opts=ide.ide_common.ide_via_ya_make_opts() + + [ + CLionOptions(), + TidyOptions(), + core.common_opts.YaBin3Options(), + ], + ) + + self['idea'] = core.yarg.OptsHandler( + action=app.execute(ide.idea.do_idea), + description='Generate stub for IntelliJ IDEA', + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + ide.ide_common.IdeYaMakeOptions(), + ide.ide_common.YaExtraArgsOptions(), + IdeaOptions(), + core.common_opts.OutputStyleOptions(), + core.common_opts.CrossCompilationOptions(), + core.common_opts.PrintStatisticsOptions(), + build.build_opts.ContinueOnFailOptions(), + build.build_opts.YMakeDebugOptions(), + build.build_opts.BuildThreadsOptions(build_threads=None), + build.build_opts.DistCacheOptions(), + build.build_opts.FlagsOptions(), + build.build_opts.IgnoreRecursesOptions(), + build.build_opts.ContentUidsOptions(), + build.build_opts.ExecutorOptions(), + build.build_opts.CustomFetcherOptions(), + build.build_opts.SandboxAuthOptions(), + core.common_opts.YaBin3Options(), + ], + unknown_args_as_free=True, + ) + ide_gradle_opts = ide.ide_common.ide_minimal_opts(targets_free=True) + [ + ide.ide_common.YaExtraArgsOptions(), + GradleOptions(), + core.yarg.ShowHelpOptions(), + build.build_opts.FlagsOptions(), + build.build_opts.CustomFetcherOptions(), + build.build_opts.SandboxAuthOptions(), + core.common_opts.CrossCompilationOptions(), + build.build_opts.ToolsOptions(), + build.build_opts.BuildTypeOptions('release'), + build.build_opts.JavaSpecificOptions(), + ] + if six.PY2: + self['gradle'] = core.yarg.OptsHandler( + action=app.execute( + lambda *a, **k: None, + handler_python_major_version=3, + ), + description='Generate gradle for project', + opts=ide_gradle_opts, + visible=False, + ) + if six.PY3: + self['gradle'] = core.yarg.OptsHandler( + action=app.execute( + ide.gradle.do_gradle, + handler_python_major_version=3, + ), + description='Generate gradle for project', + opts=ide_gradle_opts, + visible=False, + ) + self['qt'] = core.yarg.OptsHandler( + action=app.execute(self._choose_qt_handler), + description='[[imp]]ya ide qt[[rst]] is deprecated, please use clangd-based tooling instead', + opts=ide.qt.QT_OPTS + [core.common_opts.YaBin3Options()], + ) + self['goland'] = core.yarg.OptsHandler( + action=app.execute(ide.goland.do_goland), + description='Generate stub for Goland', + opts=ide.ide_common.ide_via_ya_make_opts() + + [ + ide.goland.GolandOptions(), + core.common_opts.YaBin3Options(), + ], + ) + self['pycharm'] = core.yarg.OptsHandler( + action=app.execute(ide.pycharm.do_pycharm), + description='Generate PyCharm project.', + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + PycharmOptions(), + ide.ide_common.IdeYaMakeOptions(), + ide.ide_common.YaExtraArgsOptions(), + build.build_opts.DistCacheOptions(), + core.common_opts.YaBin3Options(), + ], + visible=(pm.my_platform() != 'win32'), + ) + self['vscode-clangd'] = core.yarg.OptsHandler( + action=app.execute(ide.vscode_clangd.gen_vscode_workspace), + description=get_description('Generate VSCode clangd C++ project.', ref_name='c'), + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + ide.vscode_clangd.VSCodeClangdOptions(), + ide.ide_common.YaExtraArgsOptions(), + bcd.CompilationDatabaseOptions(), + build.build_opts.FlagsOptions(), + build.build_opts.OutputOptions(), + build.build_opts.BuildThreadsOptions(build_threads=None), + build.build_opts.ArcPrefetchOptions(), + build.build_opts.ContentUidsOptions(), + build.build_opts.ToolsOptions(), + core.common_opts.YaBin3Options(), + ], + visible=(pm.my_platform() != 'win32'), + ) + self['vscode-go'] = core.yarg.OptsHandler( + action=app.execute(ide.vscode_go.gen_vscode_workspace), + description=get_description('Generate VSCode Go project.', ref_name='golang'), + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + ide.vscode_go.VSCodeGoOptions(), + ide.ide_common.YaExtraArgsOptions(), + build.build_opts.FlagsOptions(), + build.build_opts.BuildThreadsOptions(build_threads=None), + build.build_opts.ArcPrefetchOptions(), + build.build_opts.ContentUidsOptions(), + build.build_opts.ToolsOptions(), + core.common_opts.YaBin3Options(), + ], + ) + self['vscode-py'] = core.yarg.OptsHandler( + action=app.execute(ide.vscode_py.gen_vscode_workspace), + description=get_description('Generate VSCode Python project.', ref_name='python'), + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + ide.vscode_py.VSCodePyOptions(), + ide.ide_common.YaExtraArgsOptions(), + build.build_opts.FlagsOptions(), + build.build_opts.BuildThreadsOptions(build_threads=None), + build.build_opts.ArcPrefetchOptions(), + build.build_opts.ContentUidsOptions(), + build.build_opts.ToolsOptions(), + core.common_opts.YaBin3Options(), + ], + visible=(pm.my_platform() != 'win32'), + ) + self['vscode-ts'] = core.yarg.OptsHandler( + action=app.execute(ide.vscode_py.gen_vscode_workspace), + description=get_description('Generate VSCode TypeScript project.', ref_name='typescript'), + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + ide.vscode_py.VSCodePyOptions(), + ide.ide_common.YaExtraArgsOptions(), + build.build_opts.FlagsOptions(), + build.build_opts.BuildThreadsOptions(build_threads=None), + build.build_opts.ArcPrefetchOptions(), + build.build_opts.ContentUidsOptions(), + build.build_opts.ToolsOptions(), + core.common_opts.YaBin3Options(), + ], + visible=(pm.my_platform() != 'win32'), + ) + self['vscode'] = core.yarg.OptsHandler( + action=app.execute(ide.vscode_all.gen_vscode_workspace), + description=get_description('Generate VSCode multi-language project.', ref_name='multi'), + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + ide.vscode.opts.VSCodeAllOptions(), + ide.ide_common.YaExtraArgsOptions(), + bcd.CompilationDatabaseOptions(), + build.build_opts.ArcPrefetchOptions(prefetch=True, visible=False), + build.build_opts.FlagsOptions(), + build.build_opts.OutputOptions(), + build.build_opts.BuildThreadsOptions(build_threads=None), + build.build_opts.ContentUidsOptions(), + build.build_opts.ToolsOptions(), + core.common_opts.YaBin3Options(), + ], + ) + self['vscode-ts'] = core.yarg.OptsHandler( + action=app.execute(ide.vscode_ts.gen_vscode_workspace), + description=get_description('Generate VSCode TypeScript project.', ref_name='typescript'), + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + ide.vscode_ts.VSCodeTypeScriptOptions(), + core.common_opts.YaBin3Options(), + ], + visible=False, # FIXME: remove when ready for public release + ) + self['venv'] = core.yarg.OptsHandler( + action=app.execute(ide.venv.do_venv), + description='Create or update python venv', + opts=ide.ide_common.ide_minimal_opts(targets_free=True) + + [ + build.build_opts.BuildTypeOptions('release'), + build.build_opts.BuildThreadsOptions(build_threads=None), + build.build_opts.ExecutorOptions(), + build.build_opts.FlagsOptions(), + build.build_opts.IgnoreRecursesOptions(), + build.build_opts.RebuildOptions(), + core.common_opts.BeVerboseOptions(), + core.common_opts.CrossCompilationOptions(), + ide.ide_common.YaExtraArgsOptions(), + ide.venv.VenvOptions(), + core.common_opts.YaBin3Options(), + ], + visible=(pm.my_platform() != 'win32'), + ) + if app_config.in_house: + self['fix-jb-fsnotifier'] = core.yarg.OptsHandler( + action=app.execute(devtools.ya.ide.fsnotifier.fix_fsnotifier), + description='Replace fsnotifier for JB IDEs.', + opts=[ + devtools.ya.ide.fsnotifier.FixFsNotifierOptions(), + core.common_opts.ShowHelpOptions(), + core.common_opts.DumpDebugOptions(), + core.common_opts.AuthOptions(), + core.common_opts.YaBin3Options(), + ], + ) + + @staticmethod + def _choose_qt_handler(params): + if params.run: + ide.qt.run_qtcreator(params) + elif params.remote_host: + ide.remote_ide_qt.generate_remote_project(params) + else: + ide.qt.gen_qt_project(params) diff --git a/devtools/ya/handlers/ide/ya.make b/devtools/ya/handlers/ide/ya.make new file mode 100644 index 0000000000..c0802be535 --- /dev/null +++ b/devtools/ya/handlers/ide/ya.make @@ -0,0 +1,31 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.ide + __init__.py +) + +PEERDIR( + devtools/ya/app + devtools/ya/build/build_opts + devtools/ya/core + devtools/ya/core/config + devtools/ya/core/yarg + devtools/ya/ide + devtools/ya/yalibrary/platform_matcher +) + +IF (NOT YA_OPENSOURCE) + PEERDIR( + devtools/ya/ide/fsnotifier + ) +ENDIF() + +END() + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/java/__init__.py b/devtools/ya/handlers/java/__init__.py new file mode 100644 index 0000000000..5c4c7c44e3 --- /dev/null +++ b/devtools/ya/handlers/java/__init__.py @@ -0,0 +1,49 @@ +from __future__ import absolute_import +from . import helpers + +import app +import core.yarg +from build import build_opts +import devtools.ya.test.opts as test_opts + + +def default_options(): + return [ + build_opts.BuildTargetsOptions(with_free=True), + build_opts.BeVerboseOptions(), + build_opts.ShowHelpOptions(), + build_opts.YMakeDebugOptions(), + build_opts.YMakeBinOptions(), + build_opts.YMakeRetryOptions(), + build_opts.FlagsOptions(), + ] + + +class JavaYaHandler(core.yarg.CompositeHandler): + def __init__(self): + super(JavaYaHandler, self).__init__(description='Java build helpers') + + self['dependency-tree'] = core.yarg.OptsHandler( + action=app.execute(action=helpers.print_ymake_dep_tree), + description='Print dependency tree', + opts=default_options() + [build_opts.BuildTypeOptions('release')], + visible=True, + ) + self['classpath'] = core.yarg.OptsHandler( + action=app.execute(action=helpers.print_classpath), + description='Print classpath', + opts=default_options() + [build_opts.BuildTypeOptions('release')], + visible=True, + ) + self['test-classpath'] = core.yarg.OptsHandler( + action=app.execute(action=helpers.print_test_classpath), + description='Print run classpath for test module', + opts=default_options() + [test_opts.RunTestOptions()], + visible=True, + ) + self['find-all-paths'] = core.yarg.OptsHandler( + action=app.execute(action=helpers.find_all_paths), + description='Find all PEERDIR paths of between two targets', + opts=default_options() + [build_opts.FindPathOptions()], + visible=True, + ) diff --git a/devtools/ya/handlers/java/helpers.py b/devtools/ya/handlers/java/helpers.py new file mode 100644 index 0000000000..8a5a9cd35a --- /dev/null +++ b/devtools/ya/handlers/java/helpers.py @@ -0,0 +1,250 @@ +from __future__ import absolute_import +from __future__ import print_function +import time +import sys + +import core.yarg +import build.build_opts as build_opts +import build.graph as build_graph +import build.ya_make as ya_make +from build.build_facade import gen_managed_dep_tree, gen_targets_classpath +from exts.tmp import temp_dir +import yalibrary.formatter as yaformatter + +from jbuild.gen import base +from jbuild.gen import consts +from six.moves import map + + +def fix_windows(path): + return path.replace('\\', '/') + + +def get_java_ctx_with_tests(opts): + jopts = core.yarg.merge_opts(build_opts.ya_make_options(free_build_targets=True)).params() + jopts.dump_sources = True + jopts.create_symlinks = False + jopts.__dict__.update(opts.__dict__) + jopts.debug_options.append('x') + jopts.flags["YA_IDE_IDEA"] = "yes" + + import app_ctx # XXX: via args + + _, tests, _, ctx, _ = build_graph.build_graph_and_tests( + jopts, check=True, ev_listener=ya_make.get_print_listener(jopts, app_ctx.display), display=app_ctx.display + ) + + return ctx, tests + + +def get_java_ctx(opts): + ctx, _ = get_java_ctx_with_tests(opts) + return ctx + + +NORMAL = '[[imp]]{path}[[rst]]' +EXCLUDED = '[[unimp]]{path} (omitted because of [[c:red]]EXCLUDE[[unimp]])[[rst]]' +CONFLICT = '[[unimp]]{path} (omitted because of [[c:yellow]]conflict with {conflict_resolution}[[unimp]])[[rst]]' +MANAGED = '[[imp]]{path}[[unimp]] (replaced from [[c:blue]]{orig}[[unimp]] because of [[c:blue]]DEPENDENCY_MANAGEMENT[[unimp]])[[rst]]' +IGNORED = '[[unimp]]{path} (omitted as contrib proxy library)[[rst]]' +DUPLICATE = '[[unimp]]{path} (*)[[rst]]' +DIRECT_MANAGED = '[[imp]]{path}[[unimp]] (replaced from [[c:blue]]unversioned[[unimp]] because of [[c:blue]]DEPENDENCY_MANAGEMENT[[unimp]])[[rst]]' +DIRECT_DEFAULT = ( + '[[imp]]{path}[[unimp]] (replaced from [[c:magenta]]unversioned[[unimp]] to [[c:magenta]]default[[unimp]])[[rst]]' +) + + +def arrow_str(length): + s = '' + + if length: + s += '| ' * (length - 1) + '|-->' + + return '[[unimp]]' + s + '[[rst]]' + + +def node_str(dep, excluded=False, expanded_above=False): + if expanded_above: + return DUPLICATE.format(path=dep.path) + + elif excluded: + return EXCLUDED.format(path=dep.path) + + elif dep.omitted_for_conflict: + return CONFLICT.format(path=dep.path, conflict_resolution=base.basename_unix_path(dep.conflict_resolution_path)) + + elif dep.replaced_for_dependency_management: + return MANAGED.format(path=dep.path, orig=base.basename_unix_path(dep.old_path_for_dependency_management)) + + elif dep.is_managed: + return DIRECT_MANAGED.format(path=dep.path) + + elif dep.is_default: + return DIRECT_DEFAULT.format(path=dep.path) + + else: + return NORMAL.format(path=dep.path) + + +def graph_line(depth, dep, excluded=False, expanded_above=False): + return arrow_str(depth) + node_str(dep, excluded=excluded, expanded_above=expanded_above) + + +def raise_not_a_java_path(path): + raise WrongInputException('{} is not a java module'.format(path)) + + +class WrongInputException(Exception): + mute = True + + +def print_ymake_dep_tree(opts): + with temp_dir() as tmp: + res, evlog = gen_managed_dep_tree( + build_root=tmp, + build_type=opts.build_type, + build_targets=opts.abs_targets, + debug_options=opts.debug_options, + flags=opts.flags, + ymake_bin=opts.ymake_bin, + ) + + import app_ctx + + ev_listener = ya_make.get_print_listener(opts, app_ctx.display) + for ev in evlog: + ev_listener(ev) + + formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty()) + print(formatter.format_message(res.stdout)) + return any('Type' in ev and ev['Type'] == 'Error' for ev in evlog) + + +def print_classpath(opts): + opts.flags['TRAVERSE_RECURSE'] = 'no' + opts.flags['TRAVERSE_RECURS_FOR_TEST'] = 'no' + with temp_dir() as tmp: + res, evlog = gen_targets_classpath( + build_root=tmp, + build_type=opts.build_type, + build_targets=opts.abs_targets, + debug_options=opts.debug_options, + flags=opts.flags, + ymake_bin=opts.ymake_bin, + ) + + import app_ctx + + ev_listener = ya_make.get_print_listener(opts, app_ctx.display) + for ev in evlog: + ev_listener(ev) + + formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty()) + print(formatter.format_message(res.stdout)) + return any('Type' in ev and ev['Type'] == 'Error' for ev in evlog) + + +def print_test_classpath(opts): + opts.flags['IGNORE_JAVA_DEPENDENCIES_CONFIGURATION'] = 'yes' + opts.run_tests = 3 + ctx, tests = get_java_ctx_with_tests(opts) + remaining_roots = set(map(fix_windows, opts.rel_targets)) + + formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty()) + for test in tests: + if test.project_path not in remaining_roots: + continue + classpath = test.get_classpath() + if classpath is not None: + classpath = [item[len(consts.BUILD_ROOT) + 1 :] for item in classpath] + print(formatter.format_message("[[imp]]{}[[rst]]:\n\t{}".format(test.project_path, '\n\t'.join(classpath)))) + remaining_roots.discard(test.project_path) + + for path in remaining_roots: + print(formatter.format_message("[[imp]]{}[[rst]] is not a Java test".format(path))) + + +def iter_all_routes(ctx, src_path, dest_path, route_callback, max_dist=None): + achievable = {} + + def dest_achievable_from(path): + if path not in achievable: + achievable[path] = path == dest_path or any(dest_achievable_from(d.path) for d in ctx.by_path[path].deps) + + return achievable[path] + + stack = [] + + def visit(path): + stack.append(path) + + if path == dest_path: + route_callback(stack) + stack.pop() + return # no loops + + if max_dist is not None and len(stack) - 1 >= max_dist: + stack.pop() + return + + for dep in ctx.by_path[path].deps: + dep_path = dep.path + + if dest_achievable_from(dep_path): + visit(dep_path) + + stack.pop() + + if dest_achievable_from(src_path): + visit(src_path) + + +def print_route(route): + dist = len(route) - 1 + + if dist < 4: + s = '[[c:green]]{}[[rst]]: '.format(dist) + elif dist < 7: + s = '[[c:yellow]]{}[[rst]]: '.format(dist) + else: + s = '[[c:red]]{}[[rst]]: '.format(dist) + + s += '\n[[unimp]]-->'.join(['[[imp]]' + p for p in route]) + + return s + + +def find_all_paths(opts): + start_time = time.time() + opts.flags['IGNORE_JAVA_DEPENDENCIES_CONFIGURATION'] = 'yes' + ctx = get_java_ctx(opts) + + if len(opts.rel_targets) != 2: + raise WrongInputException( + 'Expected: <from> <to>\n' + 'Got: {}'.format(' '.join(['<' + p + '>' for p in opts.rel_targets])) + ) + + src_path = fix_windows(opts.rel_targets[0]) + dest_path = fix_windows(opts.rel_targets[1]) + + if src_path not in ctx.by_path: + raise_not_a_java_path(src_path) + + if dest_path not in ctx.by_path: + raise_not_a_java_path(dest_path) + + count = [0] + + formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty()) + + def route_callback(route): + print(formatter.format_message(print_route(route))) + count[0] += 1 + + try: + iter_all_routes(ctx, src_path, dest_path, route_callback, max_dist=opts.max_dist) + finally: + report = 'Found [[c:{}]]{}[[rst]] paths in [[c:yellow]]{}[[rst]] seconds'.format( + 'green' if count[0] else 'red', count[0], time.time() - start_time + ) + print(formatter.format_message(report)) diff --git a/devtools/ya/handlers/java/ya.make b/devtools/ya/handlers/java/ya.make new file mode 100644 index 0000000000..96140c1bf1 --- /dev/null +++ b/devtools/ya/handlers/java/ya.make @@ -0,0 +1,24 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.java + __init__.py + helpers.py +) + +PEERDIR( + devtools/ya/app + devtools/ya/core/yarg + devtools/ya/build + devtools/ya/build/build_opts + devtools/ya/test/opts +) + +END() + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/make/__init__.py b/devtools/ya/handlers/make/__init__.py new file mode 100644 index 0000000000..e9d774f562 --- /dev/null +++ b/devtools/ya/handlers/make/__init__.py @@ -0,0 +1,32 @@ +from __future__ import absolute_import +from build.build_handler import do_ya_make +from build.build_opts import ya_make_options + +import core.yarg + +import app + + +class MakeYaHandler(core.yarg.OptsHandler): + description = 'Build and run tests\nTo see more help use [[imp]]-hh[[rst]]/[[imp]]-hhh[[rst]]' + + def __init__(self): + core.yarg.OptsHandler.__init__( + self, + action=app.execute(action=do_ya_make), + examples=[ + core.yarg.UsageExample('{prefix} -r', 'Build current directory in release mode', good_looking=100), + core.yarg.UsageExample( + '{prefix} -t -j16 library', 'Build and test library with 16 threads', good_looking=99 + ), + core.yarg.UsageExample( + '{prefix} --checkout -j0', 'Checkout absent directories without build', good_looking=98 + ), + ], + description=self.description, + opts=ya_make_options( + free_build_targets=True, + strip_idle_build_results=True, + ), + visible=True, + ) diff --git a/devtools/ya/handlers/make/ya.make b/devtools/ya/handlers/make/ya.make new file mode 100644 index 0000000000..275750e1f7 --- /dev/null +++ b/devtools/ya/handlers/make/ya.make @@ -0,0 +1,26 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.make + __init__.py +) + +PEERDIR( + devtools/ya/app + devtools/ya/build + devtools/ya/build/build_opts + devtools/ya/core/yarg +) + +END() + +RECURSE( + tests +) + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/maven_import/__init__.py b/devtools/ya/handlers/maven_import/__init__.py new file mode 100644 index 0000000000..f1296c0c83 --- /dev/null +++ b/devtools/ya/handlers/maven_import/__init__.py @@ -0,0 +1,31 @@ +from __future__ import absolute_import +import core.yarg +import jbuild.maven.maven_import as mi +import core.common_opts as common_opts +import build.build_opts as build_opts +import app +import app_config + +from core.yarg.help_level import HelpLevel + + +class MavenImportYaHandler(core.yarg.OptsHandler): + description = 'Import specified artifact from remote maven repository to arcadia/contrib/java.' + visible = app_config.in_house + + def __init__(self): + super(MavenImportYaHandler, self).__init__( + action=app.execute(mi.do_import), + opts=[ + build_opts.MavenImportOptions(visible=HelpLevel.BASIC), + build_opts.BuildThreadsOptions(build_threads=None), + common_opts.OutputStyleOptions(), + common_opts.TransportOptions(), + common_opts.ShowHelpOptions(), + build_opts.YMakeBinOptions(), + build_opts.CustomFetcherOptions(), + build_opts.ToolsOptions(), + ] + + build_opts.checkout_options(), + description=self.description, + ) diff --git a/devtools/ya/handlers/maven_import/ya.make b/devtools/ya/handlers/maven_import/ya.make new file mode 100644 index 0000000000..c2e16c6d35 --- /dev/null +++ b/devtools/ya/handlers/maven_import/ya.make @@ -0,0 +1,28 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.maven_import + __init__.py +) + +PEERDIR( + devtools/ya/app + devtools/ya/build/build_opts + devtools/ya/core + devtools/ya/core/yarg + devtools/ya/jbuild + devtools/ya/yalibrary/tools +) + +END() + +RECURSE( + tests +) + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/package/__init__.py b/devtools/ya/handlers/package/__init__.py new file mode 100644 index 0000000000..adaf2980f1 --- /dev/null +++ b/devtools/ya/handlers/package/__init__.py @@ -0,0 +1,88 @@ +from __future__ import absolute_import +import logging + +import app +import core.yarg +import core.common_opts +import build.ya_make +import build.targets_deref +import build.build_opts +import devtools.ya.test.opts as test_opts + +import package.docker +import package.packager +import devtools.ya.handlers.package.opts as package_opts +from core.yarg.help_level import HelpLevel + +logger = logging.getLogger(__name__) + + +class PackageYaHandler(core.yarg.OptsHandler): + description = """Build package using json package description in the release build type by default. +For more info see https://docs.yandex-team.ru/ya-make/usage/ya_package""" + + def __init__(self): + super(PackageYaHandler, self).__init__( + action=app.execute(package.packager.do_package, respawn=app.RespawnType.OPTIONAL), + description=self.description, + examples=[ + core.yarg.UsageExample( + cmd='{prefix} <path to json description>', + description='Create tarball package from json description', + ) + ], + opts=[ + package_opts.PackageOperationalOptions(), + package_opts.PackageCustomizableOptions(), + package_opts.InterimOptions(), + core.common_opts.LogFileOptions(), + core.common_opts.EventLogFileOptions(), + build.build_opts.BuildTypeOptions('release'), + build.build_opts.BuildThreadsOptions(build_threads=None), + core.common_opts.CrossCompilationOptions(), + build.build_opts.ArcPrefetchOptions(), + build.build_opts.ContentUidsOptions(), + build.build_opts.KeepTempsOptions(), + build.build_opts.RebuildOptions(), + build.build_opts.StrictInputsOptions(), + build.build_opts.DumpReportOptions(), + build.build_opts.OutputOptions(), + build.build_opts.AuthOptions(), + build.build_opts.YMakeDumpGraphOptions(), + build.build_opts.YMakeDebugOptions(), + build.build_opts.YMakeBinOptions(), + build.build_opts.YMakeRetryOptions(), + build.build_opts.ExecutorOptions(), + build.build_opts.ForceDependsOptions(), + build.build_opts.IgnoreRecursesOptions(), + core.common_opts.CustomSourceRootOptions(), + core.common_opts.CustomBuildRootOptions(), + core.common_opts.ShowHelpOptions(), + core.common_opts.BeVerboseOptions(), + core.common_opts.HtmlDisplayOptions(), + core.common_opts.CommonUploadOptions(), + build.build_opts.SandboxUploadOptions(ssh_key_option_name="--ssh-key", visible=HelpLevel.BASIC), + build.build_opts.MDSUploadOptions(visible=HelpLevel.BASIC), + core.common_opts.TransportOptions(), + build.build_opts.CustomFetcherOptions(), + build.build_opts.DistCacheOptions(), + build.build_opts.FlagsOptions(), + build.build_opts.PGOOptions(), + test_opts.RunTestOptions(), + test_opts.DebuggingOptions(), + # strip_idle_build_results must be False to avoid removal of build nodes which are + # reachable due RECURSE and used in package, but not required for tests + test_opts.DepsOptions(strip_idle_build_results=False), + test_opts.FileReportsOptions(), + test_opts.FilteringOptions(test_size_filters=None), + test_opts.PytestOptions(), + test_opts.JUnitOptions(), + test_opts.RuntimeEnvironOptions(), + test_opts.TestToolOptions(), + test_opts.UidCalculationOptions(cache_tests=False), + core.common_opts.YaBin3Options(), + ] + + build.build_opts.distbs_options() + + build.build_opts.checkout_options() + + build.build_opts.svn_checkout_options(), + ) diff --git a/devtools/ya/handlers/package/opts/__init__.py b/devtools/ya/handlers/package/opts/__init__.py new file mode 100644 index 0000000000..e65532ef6f --- /dev/null +++ b/devtools/ya/handlers/package/opts/__init__.py @@ -0,0 +1,691 @@ +import collections +import logging + +import build.build_opts +import core.yarg +import devtools.ya.test.opts as test_opts +from devtools.ya.package import const + +logger = logging.getLogger(__name__) + + +COMMON_SUBGROUP = core.yarg.Group('Common', 1) +TAR_SUBGROUP = core.yarg.Group('Tar', 2) +DEB_SUBGROUP = core.yarg.Group('Debian', 3) +DOCKER_SUBGROUP = core.yarg.Group('Docker', 4) +AAR_SUBGROUP = core.yarg.Group('Aar', 5) +RPM_SUBGROUP = core.yarg.Group('Rpm', 6) +NPM_SUBGROUP = core.yarg.Group('Npm', 7) +PYTHON_WHEEL_SUBGROUP = core.yarg.Group('Python wheel', 8) + + +class PackageOperationalOptions(core.yarg.Options): + def __init__(self): + self.artifactory_password_path = None + self.build_debian_scripts = False + self.build_only = False + self.change_log = None + self.cleanup = True + self.codec = None + self.convert = None + self.custom_data_root = None + self.custom_tests_data_root = None + self.debian_distribution = 'unstable' + self.debian_upload_token = None # please, do not remove, we really need it in opensource nebius ya + self.docker_no_cache = False + self.docker_push_image = False + self.docker_remote_image_version = None + self.docker_use_remote_cache = False + self.dump_build_targets = None + self.dump_inputs = None + self.ignore_fail_tests = False + self.list_codecs = False + self.nanny_release = None + self.package_output = None + self.packages = [] + self.publish_to = {} + self.raw_package_path = None + self.run_long_tests = False + self.sandbox_download_protocols = [] + self.upload = False + self.wheel_access_key_path = None + self.wheel_secret_key_path = None + + @staticmethod + def consumer(): + return [ + core.yarg.ArgConsumer( + names=['--publish-to'], + help='Publish package to the specified dist', + hook=core.yarg.DictPutHook('publish_to'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--build-only'], + hook=core.yarg.SetConstValueHook('build_only', True), + visible=False, + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--change-log'], + help='Change log text or path to the existing changelog file', + hook=core.yarg.SetValueHook('change_log'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--new'], + help='Use new ya package json format', + hook=core.yarg.SetConstValueHook('convert', False), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--old'], + help='Use old ya package json format', + hook=core.yarg.SetConstValueHook('convert', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--tests-data-root'], + help="Custom location for arcadia_tests_data dir, defaults to <source root>/../arcadia_tests_data", + hook=core.yarg.SetValueHook('custom_tests_data_root'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--data-root'], + help="Custom location for data dir, defaults to <source root>/../data", + hook=core.yarg.SetValueHook('custom_data_root'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--artifactory-password-path'], + help='Path to file with artifactory password', + hook=core.yarg.SetValueHook('artifactory_password_path'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.EnvConsumer( + 'YA_ARTIFACTORY_PASSWORD_PATH', + help='Path to file with artifactory password', + hook=core.yarg.SetValueHook('artifactory_password_path'), + ), + core.yarg.ArgConsumer( + names=['--dump-arcadia-inputs'], + help='Only dump inputs, do not build package', + hook=core.yarg.SetValueHook('dump_inputs'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--ignore-fail-tests'], + help='Create package, no matter tests failed or not', + hook=core.yarg.SetConstValueHook('ignore_fail_tests', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--codec'], + help='Codec name for uc compression', + hook=core.yarg.SetValueHook('codec'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--codecs-list'], + help='Show available codecs for --uc', + hook=core.yarg.SetConstValueHook('list_codecs', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + ["-O", "--package-output"], + help="Specifies directory for package output", + hook=core.yarg.SetValueHook('package_output'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.FreeArgConsumer( + help='Package description file name(s)', + hook=core.yarg.SetValueHook(name='packages'), + ), + core.yarg.ArgConsumer( + ['--sandbox-download-protocol'], + help='Sandbox download protocols comma-separated (default: http,http_tgz)', + hook=core.yarg.SetValueHook( + 'sandbox_download_protocols', transform=lambda val: [_f for _f in val.split(",") if _f] + ), + visible=False, + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--wheel-repo-access-key'], + help='Path to access key for wheel repository', + hook=core.yarg.SetValueHook('wheel_access_key_path'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=PYTHON_WHEEL_SUBGROUP, + ), + core.yarg.EnvConsumer( + 'YA_WHEEL_REPO_ACCESS_KEY_PATH', + help='Path to access key for wheel repository', + hook=core.yarg.SetValueHook('wheel_access_key_path'), + ), + core.yarg.ArgConsumer( + names=['--wheel-repo-secret-key'], + help='Path to secret key for wheel repository', + hook=core.yarg.SetValueHook('wheel_secret_key_path'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=PYTHON_WHEEL_SUBGROUP, + ), + core.yarg.EnvConsumer( + 'YA_WHEEL_SECRET_KEY_PATH', + help='Path to secret key for wheel repository', + hook=core.yarg.SetValueHook('wheel_secret_key_path'), + ), + core.yarg.ArgConsumer( + names=['--raw-package-path'], + help="Custom path for raw-package (implies --raw-package)", + hook=core.yarg.SetValueHook('raw_package_path'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--no-cleanup'], + help='Do not clean the temporary directory', + hook=core.yarg.SetConstValueHook('cleanup', False), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--build-debian-scripts'], + hook=core.yarg.SetConstValueHook('build_debian_scripts', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--debian-distribution'], + help='Debian distribution', + hook=core.yarg.SetValueHook('debian_distribution'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.EnvConsumer( + 'YA_DEBIAN_UPLOAD_TOKEN', + help='Iam token or path to iam token for nebiuscloud debian repository', + hook=core.yarg.SetValueHook('debian_upload_token'), + ), + core.yarg.ArgConsumer( + names=['--docker-push'], + help='Push docker image to registry', + hook=core.yarg.SetConstValueHook('docker_push_image', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-no-cache'], + help='Disable docker cache', + hook=core.yarg.SetConstValueHook('docker_no_cache', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ConfigConsumer("docker_no_cache"), + core.yarg.ArgConsumer( + names=['--dump-build-targets'], + hook=core.yarg.SetValueHook('dump_build_targets'), + visible=False, + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-use-remote-cache'], + help='Use image from registry as cache source', + hook=core.yarg.SetConstValueHook('docker_use_remote_cache', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-remote-image-version'], + help='Specify image version to be used as cache source', + hook=core.yarg.SetValueHook('docker_remote_image_version'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--nanny-release'], + help='Notify nanny about new release', + hook=core.yarg.SetValueHook( + 'nanny_release', transform=lambda s: s.upper(), values=const.NANNY_RELEASE_TYPES + ), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + ['--upload'], + help='Upload created package to sandbox', + hook=core.yarg.SetConstValueHook('upload', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + ] + + def postprocess(self): + if self.convert is not None: + logger.warning("Package format will be detected automatically, no need to use --new and --old") + if self.nanny_release and not self.docker_push_image: + raise core.yarg.ArgsValidatingException("Using --nanny-release without --docker-push is pointless") + + def postprocess2(self, params): + if params.raw_package_path and not params.raw_package: + params.raw_package = True + # old params compatibility + if getattr(params, 'run_long_tests', False): + params.run_tests = test_opts.RunTestOptions.RunAllTests + + +class PackageCustomizableOptions(core.yarg.Options): + """ + Don't add parameters here by default, otherwise user could use them in package.json. + For more info see https://docs.yandex-team.ru/ya-make/usage/ya_package/json#params + """ + + deb_compression_levels = collections.OrderedDict( + sorted( + { + 'none': 0, + 'low': 3, + 'medium': 6, + 'high': 9, + }.items(), + key=lambda i: i[1], + ) + ) + + def __init__(self): + self.arch_all = False + self.artifactory = None + self.compress_archive = True + self.compression_filter = None + self.compression_level = None + self.create_dbg = False + self.custom_version = None + self.debian_arch = None + self.debian_compression_level = None + self.debian_compression_type = 'gzip' + self.docker_add_host = [] + self.docker_build_arg = {} + self.docker_build_network = None + self.docker_platform = None + self.docker_registry = "registry.yandex.net" + self.docker_repository = "" + self.docker_save_image = False + self.docker_secrets = [] + self.docker_target = None + self.dupload_max_attempts = 1 + self.dupload_no_mail = False + self.ensure_package_published = False + self.force_dupload = False + self.format = None + self.full_strip = False + self.key = None + self.overwrite_read_only_files = False + self.raw_package = False + self.resource_attrs = {} + self.resource_type = "YA_PACKAGE" + self.sandbox_task_id = 0 + self.sign = True + self.sloppy_deb = False + self.store_debian = True + self.strip = False + self.wheel_platform = "" + self.wheel_python3 = False + + @staticmethod + def consumer(): + return [ + core.yarg.ArgConsumer( + names=['--strip'], + help='Strip binaries (only debug symbols: "strip -g")', + hook=core.yarg.SetConstValueHook('strip', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--full-strip'], + help='Strip binaries', + hook=core.yarg.SetConstValueHook('full_strip', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--set-sandbox-task-id'], + visible=False, + help='Use the provided task id for the package version if needed', + hook=core.yarg.SetValueHook('sandbox_task_id', int), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--wheel-platform'], + visible=True, + help='Set wheel package platform', + hook=core.yarg.SetValueHook('wheel_platform'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=PYTHON_WHEEL_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--key'], + help='The key to use for signing', + hook=core.yarg.SetValueHook('key'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--debian'], + help='Build debian package', + hook=core.yarg.SetConstValueHook('format', const.PackageFormat.DEBIAN), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--tar'], + help='Build tarball package', + hook=core.yarg.SetConstValueHook('format', const.PackageFormat.TAR), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--no-compression'], + help="Don't compress tar archive (for --tar only)", + hook=core.yarg.SetConstValueHook('compress_archive', False), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--create-dbg'], + help='Create separate package with debug info (works only in case of --strip or --full-strip)', + hook=core.yarg.SetConstValueHook('create_dbg', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + ["--compression-filter"], + help="Specifies compression filter (gzip/zstd)", + hook=core.yarg.SetValueHook('compression_filter'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + ["--compression-level"], + help="Specifies compression level (0-9 for gzip [6 is default], 0-22 for zstd [3 is default])", + hook=core.yarg.SetValueHook('compression_level', transform=lambda s: int(s)), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker'], + help='Build docker', + hook=core.yarg.SetConstValueHook('format', const.PackageFormat.DOCKER), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--rpm'], + help='Build rpm package', + hook=core.yarg.SetConstValueHook('format', const.PackageFormat.RPM), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=RPM_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--aar'], + help='Build aar package', + hook=core.yarg.SetConstValueHook('format', const.PackageFormat.AAR), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=AAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--npm'], + help='Build npm package', + hook=core.yarg.SetConstValueHook('format', const.PackageFormat.NPM), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=NPM_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--wheel'], + help='Build wheel package', + hook=core.yarg.SetConstValueHook('format', const.PackageFormat.WHEEL), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=PYTHON_WHEEL_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--wheel-python3'], + help='use python3 when building wheel package', + hook=core.yarg.SetConstValueHook('wheel_python3', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=PYTHON_WHEEL_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--artifactory'], + help='Build package and upload it to artifactory', + hook=core.yarg.SetConstValueHook("artifactory", True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-add-host'], + help='Docker --add-host', + hook=core.yarg.SetAppendHook('docker_add_host'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-secret'], + help='Same as Docker --secret. You can pass few secrets at the same time', + hook=core.yarg.SetAppendHook('docker_secrets'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-registry'], + help='Docker registry', + hook=core.yarg.SetValueHook('docker_registry'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-repository'], + help='Specify private repository', + hook=core.yarg.SetValueHook('docker_repository'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-save-image'], + help='Save docker image to archive', + hook=core.yarg.SetConstValueHook('docker_save_image', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-network'], + help='--network parameter for `docker build` command', + hook=core.yarg.SetValueHook('docker_build_network'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-platform'], + help='Specify platform for docker build (require buildx)', + hook=core.yarg.SetValueHook('docker_platform'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-build-arg'], + help='--build-arg parameter for `docker build` command, set it in the <key>=<value> form', + hook=core.yarg.DictPutHook('docker_build_arg'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--docker-target'], + help='Specifying target build stage (--target)', + hook=core.yarg.SetValueHook('docker_target'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DOCKER_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--raw-package'], + help="Used with --tar to get package content without tarring", + hook=core.yarg.SetConstValueHook('raw_package', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=TAR_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--sloppy-and-fast-debian'], + help="Fewer checks and no compression when building debian package", + hook=core.yarg.SetConstValueHook('sloppy_deb', True), + visible=False, + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--not-sign-debian'], + help='Do not sign debian package', + hook=core.yarg.SetConstValueHook('sign', False), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--custom-version'], + help='Custom package version', + hook=core.yarg.SetValueHook('custom_version'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--debian-arch'], + help='Debian arch (passed to debuild as `-a`)', + hook=core.yarg.SetValueHook('debian_arch'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--arch-all'], + help='Use "Architecture: all" in debian', + hook=core.yarg.SetConstValueHook('arch_all', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--force-dupload'], + help='dupload --force', + hook=core.yarg.SetConstValueHook('force_dupload', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['-z', '--debian-compression'], + help="deb-file compresson level ({})".format( + ", ".join(list(PackageCustomizableOptions.deb_compression_levels.keys())) + ), + hook=core.yarg.SetValueHook( + 'debian_compression_level', values=list(PackageCustomizableOptions.deb_compression_levels.keys()) + ), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['-Z', '--debian-compression-type'], + help="deb-file compression type used when building deb-file (allowed types: {}, gzip (default), xz, bzip2, lzma, none)".format( + const.DEBIAN_HOST_DEFAULT_COMPRESSION_LEVEL + ), + hook=core.yarg.SetValueHook('debian_compression_type'), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--dont-store-debian'], + help="Save debian package in a separate archive", + hook=core.yarg.SetConstValueHook('store_debian', False), + visible=False, + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + ['--upload-resource-type'], + help='Created resource type', + hook=core.yarg.SetValueHook('resource_type'), + group=build.build_opts.SANDBOX_UPLOAD_OPT_GROUP, + ), + core.yarg.ArgConsumer( + ['--upload-resource-attr'], + help='Resource attr, set it in the <name>=<value> form', + hook=core.yarg.DictPutHook(name='resource_attrs'), + group=build.build_opts.SANDBOX_UPLOAD_OPT_GROUP, + ), + core.yarg.ArgConsumer( + names=['--dupload-max-attempts'], + help='How many times try to run dupload if it fails', + hook=core.yarg.SetValueHook('dupload_max_attempts', int), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--dupload-no-mail'], + help='dupload --no-mail', + hook=core.yarg.SetConstValueHook('dupload_no_mail', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=DEB_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--overwrite-read-only-files'], + help='Overwrite read-only files in package', + hook=core.yarg.SetConstValueHook('overwrite_read_only_files', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + core.yarg.ArgConsumer( + names=['--ensure-package-published'], + help='Ensure that package is available in the repository', + hook=core.yarg.SetConstValueHook('ensure_package_published', True), + group=core.yarg.PACKAGE_OPT_GROUP, + subgroup=COMMON_SUBGROUP, + ), + ] + + def postprocess(self): + if self.debian_compression_level is not None: + self.debian_compression_level = self.deb_compression_levels[self.debian_compression_level] + if self.create_dbg: + if not self.full_strip: + self.strip = True + if self.compression_filter not in (None, 'gzip', 'zstd'): + raise core.yarg.ArgsValidatingException( + "Using unsupported compression filter: {}".format(self.compression_filter) + ) + + +class InterimOptions(core.yarg.Options): + Visible = False + + def __init__(self): + self.verify_patterns_usage = True + + # All this options + # - !! should never be available in YA_PACKAGE sandbox task !! + # - will be removed when work is done + def consumer(self): + return [ + core.yarg.ArgConsumer( + names=['--fixme-CHEMODAN-80080'], + help='See CHEMODAN-80080 and DEVTOOLSSUPPORT-12411 for more info', + hook=core.yarg.SetConstValueHook('verify_patterns_usage', False), + visible=self.Visible, + ), + ] diff --git a/devtools/ya/handlers/package/opts/ya.make b/devtools/ya/handlers/package/opts/ya.make new file mode 100644 index 0000000000..f13c20f4b5 --- /dev/null +++ b/devtools/ya/handlers/package/opts/ya.make @@ -0,0 +1,19 @@ +PY23_LIBRARY() + +PY_SRCS( + __init__.py +) + +PEERDIR( + devtools/ya/build/build_opts + devtools/ya/core + devtools/ya/core/yarg + devtools/ya/exts + devtools/ya/package/const + devtools/ya/test/const + devtools/ya/test/opts +) + +STYLE_PYTHON() + +END() diff --git a/devtools/ya/handlers/package/ya.make b/devtools/ya/handlers/package/ya.make new file mode 100644 index 0000000000..98c8a47dd9 --- /dev/null +++ b/devtools/ya/handlers/package/ya.make @@ -0,0 +1,27 @@ +PY23_LIBRARY() + +PY_SRCS( + NAMESPACE handlers.package + __init__.py +) + +PEERDIR( + contrib/python/pathlib2 + devtools/ya/app + devtools/ya/build + devtools/ya/build/build_opts + devtools/ya/core + devtools/ya/core/yarg + devtools/ya/handlers/package/opts + devtools/ya/package + devtools/ya/test/opts +) + +STYLE_PYTHON() + +END() + +RECURSE_FOR_TESTS( + bin + opts +) diff --git a/devtools/ya/handlers/test/__init__.py b/devtools/ya/handlers/test/__init__.py new file mode 100644 index 0000000000..2b3b2e4de4 --- /dev/null +++ b/devtools/ya/handlers/test/__init__.py @@ -0,0 +1,40 @@ +import app + +from build.build_handler import do_ya_make +from build.build_opts import ya_make_options + +import core.yarg + + +class YaTestYaHandler(core.yarg.OptsHandler): + description = 'Build and run all tests\n[[imp]]ya test[[rst]] is alias for [[imp]]ya make -A[[rst]]' + + def __init__(self): + core.yarg.OptsHandler.__init__( + self, + action=app.execute(action=do_ya_make), + examples=[ + core.yarg.UsageExample( + '{prefix}', + 'Build and run all tests', + good_looking=101, + ), + core.yarg.UsageExample( + '{prefix} -t', + 'Build and run small tests only', + good_looking=102, + ), + core.yarg.UsageExample( + '{prefix} -tt', + 'Build and run small and medium tests', + good_looking=103, + ), + ], + description=self.description, + opts=ya_make_options( + free_build_targets=True, + is_ya_test=True, + strip_idle_build_results=True, + ), + visible=True, + ) diff --git a/devtools/ya/handlers/test/ya.make b/devtools/ya/handlers/test/ya.make new file mode 100644 index 0000000000..70e9dee8a1 --- /dev/null +++ b/devtools/ya/handlers/test/ya.make @@ -0,0 +1,21 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.test + __init__.py +) + +PEERDIR( + devtools/ya/app + devtools/ya/build + devtools/ya/build/build_opts +) + +END() + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/tool/__init__.py b/devtools/ya/handlers/tool/__init__.py new file mode 100644 index 0000000000..b4ac4ccdca --- /dev/null +++ b/devtools/ya/handlers/tool/__init__.py @@ -0,0 +1,281 @@ +from __future__ import absolute_import +from __future__ import print_function +import os +import sys +import logging + +from core.common_opts import CrossCompilationOptions +from core.yarg import ( + ArgConsumer, + CompositeHandler, + EnvConsumer, + SetConstValueHook, + SetValueHook, + Options, + OptsHandler, + FreeArgConsumer, + ConfigConsumer, + ExtendHook, + ShowHelpException, + SetAppendHook, + BaseHook, +) + +import app + +from build.build_opts import CustomFetcherOptions, SandboxAuthOptions, ToolsOptions +from core.yarg.groups import PRINT_CONTROL_GROUP +from core.yarg.help_level import HelpLevel +from yalibrary.tools import environ, param, resource_id, task_id, tool, tools, toolchain_root, toolchain_sys_libs +from yalibrary.toolscache import lock_resource +from yalibrary.platform_matcher import is_darwin_arm64 +import core.config +import core.respawn +import exts.process +import exts.windows + +logger = logging.getLogger(__name__) + + +class ToolYaHandler(CompositeHandler): + description = 'Execute specific tool' + + @staticmethod + def common_download_options(): + return [SandboxAuthOptions(), CustomFetcherOptions(), ToolsOptions()] + + def __init__(self): + CompositeHandler.__init__( + self, + description=self.description, + examples=[ + core.yarg.UsageExample('{prefix} --ya-help', 'Print yatool specific options', good_looking=20), + core.yarg.UsageExample('{prefix} --print-path', 'Print path to tool executable file', good_looking=10), + core.yarg.UsageExample( + '{prefix} --force-update', + 'Check tool for updates before the update interval elapses', + good_looking=10, + ), + ], + ) + for x in tools(): + self[x.name] = OptsHandler( + action=app.execute(action=do_tool, respawn=app.RespawnType.OPTIONAL), + description=x.description, + visible=x.visible, + opts=[ToolOptions(x.name)] + self.common_download_options(), + unknown_args_as_free=True, + ) + + +class DummyHook(BaseHook): + def __call__(self, to, *args): + # type: ("Options", tp.Optional[tp.Any]) -> None + pass + + @staticmethod + def need_value(): + return False + + +class ToolOptions(Options): + def __init__(self, tool): + Options.__init__(self) + self.tool = tool + self.print_path = None + self.print_toolchain_path = None + self.print_toolchain_sys_libs = None + self.toolchain = None + self.param = None + self.platform = None + self.target_platforms = [] + self.need_task_id = None + self.need_resource_id = None + self.show_help = False + self.tail_args = [] + self.host_platform = None + self.hide_arm64_host_warning = False + self.force_update = False + self.force_refetch = False + + @staticmethod + def consumer(): + return [ + ArgConsumer( + ['--print-path'], + help='Only print path to tool, do not execute', + hook=SetConstValueHook('print_path', True), + ), + ArgConsumer( + ['--print-toolchain-path'], + help='Print path to toolchain root', + hook=SetConstValueHook('print_toolchain_path', True), + ), + ArgConsumer( + ['--print-toolchain-sys-libs'], + help='Print pathes to toolchsin system libraries', + hook=SetConstValueHook('print_toolchain_sys_libs', True), + ), + ArgConsumer(['--platform'], help="Set specific platform", hook=SetValueHook('platform')), + ArgConsumer(['--host-platform'], help="Set host platform", hook=SetValueHook('host_platform')), + EnvConsumer('YA_TOOL_HOST_PLATFORM', hook=SetValueHook('host_platform')), + ArgConsumer(['--toolchain'], help="Specify toolchain", hook=SetValueHook('toolchain')), + ArgConsumer(['--get-param'], help="Get specified param", hook=SetValueHook('param')), + ArgConsumer( + ['--get-resource-id'], + help="Get resource id for specific platform (the platform should be specified)", + hook=SetConstValueHook('need_resource_id', True), + ), + ArgConsumer(['--get-task-id'], help="Get task id", hook=SetConstValueHook('need_task_id', True)), + ArgConsumer(['--ya-help'], help="Show help", hook=SetConstValueHook('show_help', True)), + ArgConsumer( + ['--target-platform'], + help='Target platform', + hook=SetAppendHook('target_platforms', values=CrossCompilationOptions.generate_target_platforms_cxx()), + ), + ArgConsumer( + ['--hide-arm64-host-warning'], + help='Hide MacOS arm64 host warning', + hook=SetConstValueHook('hide_arm64_host_warning', True), + group=PRINT_CONTROL_GROUP, + visible=HelpLevel.EXPERT if is_darwin_arm64() else False, + ), + EnvConsumer('YA_TOOL_HIDE_ARM64_HOST_WARNING', hook=SetConstValueHook('hide_arm64_host_warning', True)), + ConfigConsumer('hide_arm64_host_warning'), + ArgConsumer( + ['--force-update'], + help='Check tool for updates before the update interval elapses', + hook=SetConstValueHook('force_update', True), + ), + ArgConsumer(['--force-refetch'], help='Refetch toolchain', hook=SetConstValueHook('force_refetch', True)), + ArgConsumer(['--print-fastpath-error'], help='Print fast path failure error', hook=DummyHook()), + FreeArgConsumer(help='arg', hook=ExtendHook(name='tail_args')), + ] + + def postprocess(self): + if self.show_help: + raise ShowHelpException() + if self.toolchain and self.target_platforms: + raise core.yarg.ArgsValidatingException("Do not use --toolchain and --target-platform args together") + if self.force_update: + os.environ['YA_TOOL_FORCE_UPDATE'] = "1" + + +def _replace(s, transformations): + for k, v in transformations.items(): + s = s.replace('$({})'.format(k), v) + return s + + +def _useful_env_vars(): + return {'YA_TOOL': sys.argv[0]} + + +def do_tool(params): + tool_name = params.tool + extra_args = params.tail_args + target_platform = params.target_platforms + host_platform = params.host_platform + if target_platform: + if len(target_platform) > 1: + raise Exception('Multiple target platforms are not supported by this code for now') + target_platform = target_platform[0] + else: + target_platform = None + + if is_darwin_arm64() and not host_platform: + host_platform = 'darwin' + if not params.hide_arm64_host_warning: + try: + import app_ctx + + app_ctx.display.emit_message("You use x86_64 version of selected tool.") + except Exception as e: + logger.error("Can't print arm64 warning message: {}".format(e)) + + tool_path = tool( + tool_name, + params.toolchain, + target_platform=target_platform, + for_platform=host_platform, + force_refetch=params.force_refetch, + ) + if exts.windows.on_win() and not tool_path.endswith('.exe'): # XXX: hack. Think about ya.conf.json format + logger.debug('Rename tool for win: %s', tool_path) + tool_path += '.exe' + + lock_result = False + for_platform = params.platform or params.host_platform or None + + if params.need_task_id: + tid = task_id(tool_name, params.toolchain) + if tid is not None: + print(tid) + else: + raise Exception("Tool '{}' has no task id".format(tool_name)) + elif params.need_resource_id: + print(resource_id(tool_name, params.toolchain, for_platform)) + elif params.param: + print(param(tool_name, params.toolchain, params.param)) + elif params.print_toolchain_path: + print(toolchain_root(tool_name, params.toolchain, for_platform)) + lock_result = True + elif params.print_toolchain_sys_libs: + print(toolchain_sys_libs(tool_name, params.toolchain, for_platform)) + lock_result = True + elif params.print_path: + print(tool_path) + lock_result = True + elif os.path.isfile(tool_path): + env = core.respawn.filter_env(os.environ.copy()) + + # Remove environment variables set by 'ya' wrapper. + # They are actually one-time ya-bin parameters rather than inheritable environment + # for all descendant processes. + for key in ('YA_SOURCE_ROOT', 'YA_PYVER_REQUIRE', 'YA_PYVER_SET_FORCED'): + env.pop(key, None) + + env.update(_useful_env_vars()) + for key, value in environ(tool_name, params.toolchain).items(): + env[key] = _replace( + os.pathsep.join(value), {'ROOT': toolchain_root(tool_name, params.toolchain, for_platform)} + ) + if tool_name == 'gdb': + # gdb does not fit in 8 MB stack with large cores (DEVTOOLS-5040). + try: + import resource as r + except ImportError: + pass + else: + soft, hard = r.getrlimit(r.RLIMIT_STACK) + new = 128 << 20 + logger.debug("Limit info: soft=%d hard=%d new=%d", soft, hard, new) + if hard != r.RLIM_INFINITY: + new = min(new, hard) + logger.debug("Limit info: new=%d", new) + if new > soft: + logger.debug("Limit info: setting new limits=(%d, %d)", new, hard) + try: + r.setrlimit(r.RLIMIT_STACK, (new, hard)) + except ValueError as e: + logger.error("Failure while setting RLIMIT_STACK ({}, {}), {}".format(new, hard, e)) + logger.exception("While setting RLIMIT_STACK") + arc_root = core.config.find_root(fail_on_error=False) + if arc_root is not None: + logger.debug('Arcadia root found: %s', arc_root) + extra_args = ['-ex', 'set substitute-path /-S/ {}/'.format(arc_root)] + extra_args + extra_args = ['-ex', 'set filename-display absolute'] + extra_args + if ( + tool_name == 'arc' + and params.username not in {'sandbox', 'root'} + and os.getenv('YA_ALLOW_TOOL_ARC', 'no') != 'yes' + ): + message = ( + 'Please, use natively installed arc, install guide:' + ' https://docs.yandex-team.ru/devtools/intro/quick-start-guide#arc-setup' + ) + raise core.yarg.ArgsValidatingException(message) + exts.process.execve(tool_path, extra_args, env=env) + + if lock_result: + lock_resource(toolchain_root(tool_name, params.toolchain, for_platform)) diff --git a/devtools/ya/handlers/tool/ya.make b/devtools/ya/handlers/tool/ya.make new file mode 100644 index 0000000000..6021f3ea29 --- /dev/null +++ b/devtools/ya/handlers/tool/ya.make @@ -0,0 +1,33 @@ +PY23_LIBRARY() + +STYLE_PYTHON() + +PY_SRCS( + NAMESPACE handlers.tool + __init__.py +) + +PEERDIR( + devtools/ya/app + devtools/ya/build/build_opts + devtools/ya/core + devtools/ya/core/config + devtools/ya/core/respawn + devtools/ya/core/yarg + devtools/ya/exts + devtools/ya/yalibrary/platform_matcher + devtools/ya/yalibrary/tools + devtools/ya/yalibrary/toolscache + library/python/windows +) + +END() + +RECURSE( + tests +) + +RECURSE_FOR_TESTS( + bin + tests +) diff --git a/devtools/ya/handlers/ya.make b/devtools/ya/handlers/ya.make new file mode 100644 index 0000000000..d88914356d --- /dev/null +++ b/devtools/ya/handlers/ya.make @@ -0,0 +1,151 @@ +PY23_LIBRARY(ya-lib) + +SRCDIR(devtools/ya) + +PEERDIR( + devtools/ya/handlers/make + devtools/ya/handlers/package + devtools/ya/handlers/test + devtools/ya/handlers/tool + devtools/ya/handlers/ide + devtools/ya/handlers/dump + devtools/ya/handlers/gc + devtools/ya/handlers/gen_config + devtools/ya/handlers/maven_import + devtools/ya/handlers/java + # devtools/ya/handlers/analyze_make +) + +IF (NOT YA_OPENSOURCE) + PEERDIR( + devtools/ya/handlers/__trace__ + devtools/ya/handlers/addremove + devtools/ya/handlers/analyze_make + devtools/ya/handlers/buf + devtools/ya/handlers/check + devtools/ya/handlers/clang_tidy + devtools/ya/handlers/clone + devtools/ya/handlers/completion + devtools/ya/handlers/dctl + devtools/ya/handlers/download + devtools/ya/handlers/exec + devtools/ya/handlers/fetch + devtools/ya/handlers/fix_includes + devtools/ya/handlers/krevedko + devtools/ya/handlers/notify + devtools/ya/handlers/paste + devtools/ya/handlers/pr + devtools/ya/handlers/profile + devtools/ya/handlers/project + devtools/ya/handlers/py + devtools/ya/handlers/remote_gdb + devtools/ya/handlers/repo_check + devtools/ya/handlers/shell + devtools/ya/handlers/shelve + devtools/ya/handlers/stat + devtools/ya/handlers/style + devtools/ya/handlers/svn + devtools/ya/handlers/unshelve + devtools/ya/handlers/upload + devtools/ya/handlers/vmctl + devtools/ya/handlers/webide + devtools/ya/handlers/whoami + devtools/ya/handlers/wine + devtools/ya/handlers/yav + + devtools/ya/handlers/py23migration/_ya0bin2 + devtools/ya/handlers/py23migration/_ya0bin3 + devtools/ya/handlers/py23migration/_ya2bin0 + devtools/ya/handlers/py23migration/_ya2bin2 + devtools/ya/handlers/py23migration/_ya3bin0 + devtools/ya/handlers/py23migration/_ya3bin3 + ) + IF (PYTHON3) + PEERDIR( + devtools/ya/handlers/py23migration/py23_utils + devtools/ya/handlers/vim + devtools/ya/handlers/curl + devtools/ya/handlers/neovim + devtools/ya/handlers/gdb + devtools/ya/handlers/emacs + devtools/ya/handlers/grep + devtools/ya/handlers/jstyle + devtools/ya/handlers/nile + devtools/ya/handlers/sed + devtools/ya/handlers/ydb + devtools/ya/handlers/yql + ) + ENDIF() +ENDIF() +END() + +RECURSE( + __trace__ + addremove + analyze_make + autocheck + buf + check + clang_tidy + clone + completion + curl + dctl + download + dump + emacs + exec + fetch + fix_includes + gc + gdb + gen_config + grep + ide + java + jstyle + krevedko + make + maven_import + neovim + nile + notify + package + paste + pr + profile + project + py + remote_gdb + repo_check + sed + shell + shelve + stat + style + svn + test + tool + unshelve + upload + vim + vmctl + webide + whoami + wine + yav + ydb + yql +) + +IF (NOT OPENSOURCE) + RECURSE( + py23migration/_ya0bin2 + py23migration/_ya0bin3 + py23migration/_ya2bin0 + py23migration/_ya3bin0 + py23migration/_ya3bin3 + py23migration/_ya2bin2 + py23migration/py23_utils + ) +ENDIF() diff --git a/devtools/ya/opensource/ya.conf b/devtools/ya/opensource/ya.conf new file mode 100644 index 0000000000..fb140d072a --- /dev/null +++ b/devtools/ya/opensource/ya.conf @@ -0,0 +1,92 @@ +# Please keep this in sync with arcadia/ya.conf + +build_cache = true +build_cache_conf = ['cas_logging=true', 'graph_info=true'] +build_cache_master = true +cache_codec = '' +cache_size = 150374182400 +content_uids = true +dir_outputs = true +dir_outputs_test_mode = true +dump_debug_enabled = true +fail_maven_export_with_tests = true +incremental_build_dirs_cleanup = true +oauth_exchange_ssh_keys = true +remove_implicit_data_path = true +remove_result_node = true +tools_cache = true +tools_cache_master = true +use_atd_revisions_info = true +use_jstyle_server = true +use_command_file_in_testtool = true + +[test_tool3_handlers] +build_clang_coverage_report = true +build_go_coverage_report = true +build_python_coverage_report = false +build_sancov_coverage_report = true +build_ts_coverage_report = true +canonization_result_node = true +canonize = true +check_external = true +check_mds = true +check_resource = true +checkout = true +cov_merge_vfs = false +create_allure_report = true +download = true +list_result_node = true +list_tests = true +merge_coverage_inplace = true +merge_python_coverage = true +minimize_fuzz_corpus = true +populate_token_to_sandbox_vault = true +resolve_clang_coverage = true +resolve_go_coverage = true +resolve_java_coverage = true +resolve_python_coverage = false +resolve_sancov_coverage = true +resolve_ts_coverage = true +result_node = true +results_accumulator = true +results_merger = true +run_boost_test = true +run_check = true +run_clang_tidy = true +run_classpath_clash = true +run_coverage_extractor = true +run_custom_lint = true +run_diff_test = false +run_eslint = true +run_exectest = true +run_fuzz = true +run_fuzz_result_node = true +run_g_benchmark = true +run_go_fmt = true +run_go_test = true +run_go_vet = true +run_gtest = true +run_hermione = true +run_javastyle = true +run_jest = true +run_ktlint_test = true +run_pyimports = true +run_skipped_test = true +run_test = true +run_ut = true +run_y_benchmark = true +sandbox_run_test = true +unify_clang_coverage = true +upload = false +upload_coverage = false +ytexec_run_test = true + +# ===== opensource only table params ===== + +[host_platform_flags] +OPENSOURCE = "yes" +USE_PREBUILT_TOOLS = "no" + +[flags] +OPENSOURCE = "yes" +USE_PREBUILT_TOOLS = "no" |