diff options
| author | robot-piglet <[email protected]> | 2023-12-04 15:32:14 +0300 |
|---|---|---|
| committer | robot-piglet <[email protected]> | 2023-12-05 01:22:50 +0300 |
| commit | c21ed9eedf73010bc81342518177dfdfb0d56bd7 (patch) | |
| tree | 72f8fde4463080cfe5a38eb0babc051cfe32c51e /devtools/ya | |
| parent | ec1311bf2e8cc231723b8b5e484ca576663a1309 (diff) | |
Intermediate changes
Diffstat (limited to 'devtools/ya')
30 files changed, 0 insertions, 5080 deletions
diff --git a/devtools/ya/chameleon_bin/__main__.py b/devtools/ya/chameleon_bin/__main__.py deleted file mode 100644 index f79b940fdb0..00000000000 --- a/devtools/ya/chameleon_bin/__main__.py +++ /dev/null @@ -1,59 +0,0 @@ -import logging -import os -import shutil -from library.python.testing.recipe import declare_recipe -import yatest.common - - -class ChameleonRecipe: - def __init__(self): - self.bin2_root: str = None - self.bin3_root: str = None - self.bin2: str = None - self.bin3: str = None - - def post_init(self): - self.bin2_root = yatest.common.build_path("devtools/ya/bin") - self.bin3_root = yatest.common.build_path("devtools/ya/bin3") - self.bin2 = os.path.join(self.bin2_root, "ya-bin") - self.bin3 = os.path.join(self.bin3_root, "ya-bin") - - logging.info("ya-bin2 exists: %s", os.path.exists(self.bin2)) - logging.info("ya-bin3 exists: %s", os.path.exists(self.bin3)) - - @classmethod - def check_argv_py3(cls, argv): - return argv[0] == "yes" - - def start(self, argv): - self.post_init() - - logging.info("args: %s", argv) - - assert not (os.path.exists(self.bin2) and os.path.exists(self.bin3)), "Only one version of ya-bin can be in DEPENDS" - - if self.check_argv_py3(argv): - if not os.path.exists(self.bin2): - if not os.path.exists(self.bin2_root): - os.mkdir(self.bin2_root) - logging.debug("Create bin2 root folder: %s", self.bin2_root) - else: - logging.debug("Maybe test was restarted") - shutil.copy(self.bin3, self.bin2) - logging.debug("copy ya-bin3 to bin2 root: %s -> %s", self.bin3, self.bin2) - - - def stop(self, argv): - self.post_init() - - logging.info("args: %s", argv) - - if self.check_argv_py3(argv): - if os.path.exists(self.bin2): - logging.debug("Remove bin2 executable: %s", self.bin2) - # TOFIX v-korovin - os.remove(self.bin2) - -if __name__ == "__main__": - recipe = ChameleonRecipe() - declare_recipe(recipe.start, recipe.stop) diff --git a/devtools/ya/chameleon_bin/recipe.inc b/devtools/ya/chameleon_bin/recipe.inc deleted file mode 100644 index 4e705cc996e..00000000000 --- a/devtools/ya/chameleon_bin/recipe.inc +++ /dev/null @@ -1,32 +0,0 @@ -DEPENDS(devtools/ya/chameleon_bin) - -INCLUDE(${ARCADIA_ROOT}/devtools/ya/opensource.inc) - -IF (PYTHON3) - DEPENDS(devtools/ya/bin3) -ELSE() - IF (NOT OPENSOURCE) - DEPENDS(devtools/ya/bin) - ENDIF() -ENDIF() - -IF (NEBIUS) - DATA( - arcadia/nebius/devtools/ya-bin/ya.conf - ) -ELSEIF (OPENSOURCE) - DATA( - arcadia/devtools/ya/opensource/ya.conf - ) -ELSE() - DATA( - arcadia/ya.conf - ) -ENDIF() - -DATA( - arcadia/ya -) - -USE_RECIPE(devtools/ya/chameleon_bin/chameleon_bin ${PYTHON3}) - diff --git a/devtools/ya/chameleon_bin/ya.make b/devtools/ya/chameleon_bin/ya.make deleted file mode 100644 index 3b418c26512..00000000000 --- a/devtools/ya/chameleon_bin/ya.make +++ /dev/null @@ -1,12 +0,0 @@ -PY3_PROGRAM() - -PY_SRCS(__main__.py) - -PEERDIR( - library/python/testing/recipe - library/python/testing/yatest_common -) - -NO_LINT() - -END() diff --git a/devtools/ya/handlers/dump/__init__.py b/devtools/ya/handlers/dump/__init__.py deleted file mode 100644 index 77eb2f45d20..00000000000 --- a/devtools/ya/handlers/dump/__init__.py +++ /dev/null @@ -1,1161 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -import csv -import pickle -import exts.yjson as json -import logging -import os -import sys -import enum -import six - -from build.build_facade import ( - gen_all_loops, - gen_dir_loops, - gen_graph, - gen_licenses_list, - gen_forced_deps_list, - gen_modules, - gen_module_info, - gen_plan, - gen_json_graph, - gen_uids, - gen_srcdeps, - gen_dependencies, - gen_test_dart, - gen_conf, - gen_filelist, - gen_relation, -) -from handlers.dump.gen_conf_docs import dump_mmm_docs -from build.dir_graph import reachable, gen_dir_graph -from build.compilation_database import dump_compilation_database, COMPILATION_DATABASE_OPTS -from exts.strtobool import strtobool -from exts.tmp import temp_dir -from core.yarg import ( - CompositeHandler, - OptsHandler, - ArgConsumer, - FreeArgConsumer, - SetValueHook, - Options, - SetConstValueHook, - SetConstAppendHook, - UpdateValueHook, - SetAppendHook, - ExtendHook, - ArgsValidatingException, - Group, -) -from yalibrary.vcs import vcsversion -from core.imprint import imprint -from build.build_opts import ( - YMakeDebugOptions, - YMakeBinOptions, - FlagsOptions, - CustomFetcherOptions, - ToolsOptions, - SandboxAuthOptions, - JavaSpecificOptions, -) -from build.build_opts import ( - BuildTypeOptions, - BuildTargetsOptions, - ShowHelpOptions, - CustomBuildRootOptions, - ContinueOnFailOptions, -) -from build.build_opts import YMakeRetryOptions, ConfigurationPresetsOptions -from core.common_opts import CrossCompilationOptions, YaBin3Options -from test.explore import generate_tests_by_dart -from devtools.ya.test.dartfile import decode_recipe_cmdline - -import app -import app_config - -logger = logging.getLogger(__name__) - - -class DumpYaHandler(CompositeHandler): - common_opts = [ShowHelpOptions()] - - @staticmethod - def common_build_facade_opts(with_free_targets=True): - return [ - FlagsOptions(), - CustomFetcherOptions(), - SandboxAuthOptions(), - YMakeBinOptions(), - YMakeRetryOptions(), - CrossCompilationOptions(), - BuildTypeOptions('release'), - ContinueOnFailOptions(), - BuildTargetsOptions(with_free=with_free_targets), - ToolsOptions(), - ConfigurationPresetsOptions(), - YaBin3Options(), - ] - - def __init__(self): - CompositeHandler.__init__(self, description='Repository related information') - self['modules'] = OptsHandler( - action=app.execute(action=do_modules), - description='All modules', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DepTraverseOptions(), - DepFilterOptions(), - PeerDirectoriesOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - ], - ) - self['module-info'] = OptsHandler( - action=app.execute(action=do_module_info), - description='Modules info', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DepTraverseOptions(), - DepFilterOptions(), - PeerDirectoriesOptions(), - DataOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - DumpModuleInfoOptions(), - ], - ) - self['src-deps'] = OptsHandler( - action=app.execute(action=do_dump_srcdeps), - description='Dump of all source dependencies', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DumpSrcDepsOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - ], - ) - self['dir-graph'] = OptsHandler( - action=app.execute(action=do_dir_graph), - description='Dependencies between directories', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DumpDirOptions(), - SplitByTypeOptions(), - DumpTestListOptions(), - DepTraverseOptions(), - DepFilterOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - LegacyDepsOptions(), - ], - ) - self['dep-graph'] = OptsHandler( - action=app.execute(action=do_dep_graph), - description='Dependency internal graph', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DepGraphOutputOptions(), - DepTraverseOptions(), - DepFilterOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - LegacyDepsOptions(), - ], - ) - self['dot-graph'] = OptsHandler( - action=app.execute(action=do_dot_graph), - description='Dependency between directories in dot format', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DepTraverseOptions(), - DepFilterOptions(), - PeerDirectoriesOptions(), - CustomBuildRootOptions(), - ], - ) - self['json-dep-graph'] = OptsHandler( - action=app.execute(action=do_json_dep_graph), - description='Dependency graph as json', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DepTraverseOptions(), - DepFilterOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - LegacyDepsOptions(), - ], - ) - self['build-plan'] = OptsHandler( - action=app.execute(action=do_build_plan), - description='Build plan', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DepTraverseOptions(), - CustomBuildRootOptions(), - YMakeDebugOptions(), - ], - ) - self['compile-commands'] = OptsHandler( - action=app.execute(action=dump_compilation_database), - description='JSON compilation database', - opts=COMPILATION_DATABASE_OPTS + [ToolsOptions(), CustomFetcherOptions(), SandboxAuthOptions()], - ) - self['compilation-database'] = OptsHandler( - action=app.execute(action=dump_compilation_database), - description='Alias for compile-commands', - opts=COMPILATION_DATABASE_OPTS + [ToolsOptions(), CustomFetcherOptions(), SandboxAuthOptions()], - ) - self['relation'] = OptsHandler( - action=app.execute(action=do_relation), - description='PEERDIR relations. Please don\'t run from the arcadia root.', - opts=self.common_opts - + self.common_build_facade_opts(False) - + [ - RelationSrcDstOptions(free_dst=True), - DepTraverseOptions(), - DepFilterOptions(), - PeerDirectoriesOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - ], - ) - self['all-relations'] = OptsHandler( - action=app.execute(action=do_all_relations), - description='All relations between internal graph nodes in dot format. Please don\'t run from the arcadia root.', - opts=self.common_opts - + self.common_build_facade_opts(False) - + [ - RelationSrcDstOptions(free_dst=True), - DumpAllRelationsOptions(), - DepTraverseOptions(), - DepFilterOptions(), - PeerDirectoriesOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - ], - ) - self['files'] = OptsHandler( - action=app.execute(action=do_files), - description='File list', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - FilesFilterOptions(), - DepTraverseOptions(), - DepFilterOptions(), - DataOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - ], - ) - self['loops'] = OptsHandler( - action=app.execute(action=do_loops), - description='All loops in arcadia', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DepTraverseOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - ], - ) - self['peerdir-loops'] = OptsHandler( - action=app.execute(action=do_peerdir_loops), - description='Loops by peerdirs', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - DepTraverseOptions(), - YMakeDebugOptions(), - CustomBuildRootOptions(), - ], - ) - self['imprint'] = OptsHandler( - action=app.execute(action=do_imprint), - description='Directory imprint', - opts=self.common_opts + [CustomFetcherOptions(), SandboxAuthOptions(), ToolsOptions()], - ) - self['uids'] = OptsHandler( - action=app.execute(action=do_uids), - description='All targets uids', - opts=self.common_opts + self.common_build_facade_opts(), - visible=False, - ) - self['test-list'] = OptsHandler( - action=app.execute(action=do_test_list), - description='All test entries', - opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions()], - ) - self['json-test-list'] = OptsHandler( - action=app.execute(action=do_json_test_list), - description='All test entries as json', - opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions()], - ) - self['conf'] = OptsHandler( - action=app.execute(action=do_conf), - description='Print build conf', - opts=self.common_opts - + [ - FlagsOptions(), - CustomFetcherOptions(), - SandboxAuthOptions(), - CrossCompilationOptions(), - ToolsOptions(), - BuildTypeOptions('release'), - JavaSpecificOptions(), - ], - visible=False, - ) - self['licenses'] = OptsHandler( - action=app.execute(action=do_licenses), - description='Print known licenses grouped by their properties', - opts=self.common_opts - + self.common_build_facade_opts() - + [ - YMakeDebugOptions(), - CustomBuildRootOptions(), - DumpLicenseOptions(), - ], - visible=False, - ) - self['forced-deps'] = OptsHandler( - action=app.execute(action=do_forced_deps), - description='Print known forced dependency management', - opts=FullForcedDepsOptions(), - visible=False, - ) - self['conf-docs'] = OptsHandler( - action=app.execute(action=do_conf_docs), - description='Print descriptions of entities (modules, macros, multimodules, etc.)', - opts=self.common_opts + self.common_build_facade_opts() + [DumpDescriptionOptions()], - ) - self['root'] = OptsHandler( - action=app.execute(lambda params: sys.stdout.write(params.arc_root) and 0), - description='Print Arcadia root', - opts=self.common_opts, - ) - self['vcs-info'] = OptsHandler( - action=app.execute(action=do_dump_vcs_info), - description='Print VCS revision information.', - opts=self.common_opts + self.common_build_facade_opts(), - visible=False, - ) - self['raw-vcs-info'] = OptsHandler( - action=app.execute(action=do_dump_raw_vcs_info), - description='Print VCS revision information.', - opts=self.common_opts + self.common_build_facade_opts(), - visible=False, - ) - self['svn-revision'] = OptsHandler( - action=app.execute(action=do_dump_svn_revision), - description='Print SVN revision information.', - opts=self.common_opts + self.common_build_facade_opts(), - visible=False, - ) - self['recipes'] = OptsHandler( - action=app.execute(action=do_recipes), - description='All recipes used in tests', - opts=self.common_opts + self.common_build_facade_opts() + [DumpTestListOptions(), DumpRecipesOptions()], - ) - if app_config.in_house: - import devtools.ya.handlers.dump.arcadia_specific as arcadia_specific - from handlers.dump.debug import debug_handler - - self['groups'] = arcadia_specific.GroupsHandler() - self['atd-revisions'] = OptsHandler( - action=app.execute(action=arcadia_specific.do_atd_revisions), - description='Dump revisions of trunk/arcadia_tests_data', - opts=self.common_opts - + [DumpAtdRevisionOptions(), CustomFetcherOptions(), SandboxAuthOptions(), ToolsOptions()], - ) - self['debug'] = debug_handler - - -def _do_dump(gen_func, params, debug_options=[], write_stdout=True, build_root=None, **kwargs): - for name in 'debug_options', 'filter_opts', 'peerdir_opts': - if hasattr(params, name): - debug_options.extend(getattr(params, name)) - logger.debug('abs_targets: %s', params.abs_targets) - with temp_dir() as tmp: - res = gen_func( - build_root=build_root or tmp, - build_type=params.build_type, - build_targets=params.abs_targets, - debug_options=debug_options, - flags=params.flags, - ymake_bin=getattr(params, 'ymake_bin', None), - host_platform=params.host_platform, - target_platforms=params.target_platforms, - **kwargs - ) - if write_stdout: - sys.stdout.write(res.stdout) - return res - - -def do_modules(params): - _do_dump(gen_modules, params) - - -def do_module_info(params, write_stdout=True): - if params.with_data: - params.flags['YMAKE_ADD_DATA'] = 'yes' - return _do_dump( - gen_func=gen_module_info, - params=params, - modules_info_filter=getattr(params, 'modules_info_filter'), - modules_info_file=getattr(params, 'modules_info_file'), - write_stdout=write_stdout, - ) - - -def do_dir_graph(params): - def merge_lists(one, two): - return sorted(set(one + two)) - - def get_test_dependencies(deps, arc_root, test, test_data=None): - data = merge_lists(test_data or [], deps) - return merge_lists(data, get_canondata_paths(arc_root, test)) - - if params.dump_deps: - deps = _do_dump(gen_dependencies, params, params.legacy_deps_opts, write_stdout=False) - json.dump(json.loads(deps.stdout), sys.stdout, indent=4, sort_keys=True) - return - - if params.dump_reachable_dirs: - _do_dump(gen_graph, params, params.legacy_deps_opts + ['x', 'M']) - return - - dg = _do_dump( - gen_dir_graph, params, params.legacy_deps_opts, write_stdout=False, split_by_types=params.split_by_type - ) - - tests = get_tests(params) - for test in tests: - test_data = test.get_arcadia_test_data() - if test.project_path in test_data: - test_data.remove(test.project_path) - if params.split_by_type: - if test.project_path not in dg: - dg[test.project_path] = {} - dg[test.project_path]["INCLUDE"] = get_test_dependencies( - dg[test.project_path].get("INCLUDE", []), params.arc_root, test - ) - dg[test.project_path]["DATA"] = merge_lists(dg[test.project_path].get("DATA", []), test_data) - else: - dg[test.project_path] = get_test_dependencies( - dg.get(test.project_path, []), params.arc_root, test, test_data - ) - - if params.trace_from is not None: - print('\n'.join(reachable(dg, params.trace_from, params.split_by_type))) - - elif params.plain_dump: - - def get_plain_deps(): - plain_deps = set() - for k, v in six.iteritems(dg): - plain_deps.add(k) - plain_deps |= set(v) - return sorted(list(plain_deps)) - - json.dump(get_plain_deps(), sys.stdout, indent=4, sort_keys=True) - else: - json.dump(dg, sys.stdout, indent=4, sort_keys=True) - - -def get_canondata_paths(arc_root, test): - paths = [] - canondata_path = os.path.join(arc_root, test.project_path, "canondata") - if os.path.exists(canondata_path): - for root, dirs, files in os.walk(canondata_path): - paths.append(os.path.relpath(root, arc_root)) - return paths - - -DUMP_OPTS_GROUP = Group("Dump options", 0) - - -class SplitByTypeOptions(Options): - def __init__(self): - self.split_by_type = False - - @staticmethod - def consumer(): - return ArgConsumer( - ['--split'], - help='split by type', - hook=SetConstValueHook('split_by_type', True), - group=DUMP_OPTS_GROUP, - ) - - -class DumpDirOptions(Options): - def __init__(self): - self.trace_from = None - self.plain_dump = False - self.dump_deps = False - self.dump_reachable_dirs = False - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--trace'], - help='trace from sources', - hook=SetValueHook('trace_from'), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--plain'], - help='plain dump mode', - hook=SetConstValueHook('plain_dump', True), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--dump-deps'], - help='Dump dependencies (legacy)', - hook=SetConstValueHook('dump_deps', True), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--reachable-dirs'], - help='Dump reachable dirs', - hook=SetConstValueHook('dump_reachable_dirs', True), - group=DUMP_OPTS_GROUP, - ), - ] - - -class PeerDirectoriesOptions(Options): - def __init__(self): - self.peerdir_opts = [] - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--old-peerdirs'], - help='Include indirect (module->dir->module) PEERDIRs', - hook=SetConstAppendHook('peerdir_opts', 'I'), - group=DUMP_OPTS_GROUP, - visible=False, - ) - ] - - -class LegacyDepsOptions(Options): - def __init__(self): - self.legacy_deps_opts = ['I'] - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--no-legacy-deps'], - help='Exclude legacy dependencies (inderect PEERDIR relations)', - hook=SetConstValueHook('legacy_deps_opts', []), - group=DUMP_OPTS_GROUP, - ) - ] - - -class DumpSrcDepsOptions(Options): - def __init__(self): - self.with_yamake = False - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--with-yamakes'], - help='Include ya.make files, that are only used as a build configuration', - hook=SetConstValueHook('with_yamake', True), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DepFilterOptions(Options): - def __init__(self): - self.filter_opts = [] - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--no-tools'], - help='Exclude tools dependencies', - hook=SetConstAppendHook('filter_opts', 'V'), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--no-addincls'], - help='Exclude ADDINCLs dependencies', - hook=SetConstAppendHook('filter_opts', 'T'), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DepGraphOutputType(enum.Enum): - CUSTOM = 0 - FLAT_JSON = 1 - FLAT_JSON_FILES = 2 - - -class DepGraphOutputOptions(Options): - def __init__(self): - self.output_type = DepGraphOutputType.CUSTOM - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--flat-json'], - help='Dump dep-graph in flat json format', - hook=SetConstValueHook('output_type', DepGraphOutputType.FLAT_JSON), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--flat-json-files'], - help='Dump dep-graph in flat json format without commands', - hook=SetConstValueHook('output_type', DepGraphOutputType.FLAT_JSON_FILES), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DepTraverseOptions(Options): - @staticmethod - def consumer(): - def test_depends_updater(flags): - flags['TRAVERSE_DEPENDS'] = 'yes' - flags['TRAVERSE_RECURSE_FOR_TESTS'] = 'yes' - return flags - - def ignore_recurses_updater(flags): - flags['TRAVERSE_RECURSE'] = 'no' - return flags - - # Hooks update flags property from FlagsOptions - return [ - ArgConsumer( - ['-t', '--force-build-depends'], - help='Include DEPENDS and RECURSE_FOR_TESTS dependencies', - hook=UpdateValueHook('flags', test_depends_updater), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['-A'], - help='Same as -t', - hook=UpdateValueHook('flags', test_depends_updater), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--ignore-recurses'], - help='Exclude all RECURSE dependencies', - hook=UpdateValueHook('flags', ignore_recurses_updater), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DataOptions(Options): - def __init__(self): - self.with_data = False - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--with-data'], - help='Include DATA files and dirs', - hook=SetConstValueHook('with_data', True), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DumpTestListOptions(Options): - def __init__(self): - self.follow_dependencies = False - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--skip-deps'], - help='Now default and ignored', - hook=SetConstValueHook('follow_dependencies', False), - group=DUMP_OPTS_GROUP, - visible=False, - deprecated=True, - ), - ] - - -class DumpLicenseOptions(Options): - def __init__(self): - self.json_licenses = False - self.link_type = 'static' - self.custom_tags = [] - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--json'], - help='Machine oriented json representation', - hook=SetConstValueHook('json_licenses', True), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--link-type'], - help='Specify library link type (static|dynamic)', - hook=SetValueHook('link_type'), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--custom-license-tag'], - help='Specify library link type (static|dynamic)', - hook=SetAppendHook('custom_tags'), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DumpForcedDepsOptions(Options): - def __init__(self): - self.json_forced_deps = False - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--json'], - help='Json forced dependency representation', - hook=SetConstValueHook('json_forced_deps', True), - group=DUMP_OPTS_GROUP, - ), - ] - - -def FullForcedDepsOptions(): - return ( - DumpYaHandler.common_opts - + DumpYaHandler.common_build_facade_opts() - + [ - YMakeDebugOptions(), - CustomBuildRootOptions(), - DumpForcedDepsOptions(), - ] - ) - - -class RelationSrcDstOptions(Options): - def __init__(self, free_dst=False): - self._free_dst = free_dst - self.recursive = False - self.relation_src = [] - self.relation_dst = [] - - def consumer(self): - res = [ - ArgConsumer( - ['--from'], - help='Dump relations from this target (path relative to the arcadia root)', - hook=SetAppendHook('relation_src'), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--to'], - help='Dump relations to this target (path relative to the arcadia root)', - hook=SetAppendHook('relation_dst'), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--recursive'], - help='Show relations between RELATION_SRC and all modules from RELATION_DST directories', - hook=SetConstValueHook('recursive', True), - group=DUMP_OPTS_GROUP, - ), - ] - if self._free_dst: - res.append( - FreeArgConsumer( - help='relation destination target', - hook=ExtendHook('relation_dst'), - ) - ) - return res - - def postprocess(self): - if len(self.relation_dst) == 0: - raise ArgsValidatingException('Error: no target is set') - - -class FilesFilterOptions(Options): - def __init__(self): - self.skip_make_files = False - self.mark_make_files = False - - def consumer(self): - res = [ - ArgConsumer( - ['--skip-make-files'], - help='Skip all make files', - hook=SetConstValueHook('skip_make_files', True), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--mark-make-files'], - help='Mark all make files as Makefile', - hook=SetConstValueHook('mark_make_files', True), - group=DUMP_OPTS_GROUP, - ), - ] - return res - - -class DumpAllRelationsOptions(Options): - def __init__(self): - self.json_format = False - self.show_targets_deps = False - - def consumer(self): - res = [ - ArgConsumer( - ['--json'], - help='Dump relations in json format', - hook=SetConstValueHook('json_format', True), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--show-targets-deps'], - help='Show dependencies between RELATION_DST targets', - hook=SetConstValueHook('show_targets_deps', True), - group=DUMP_OPTS_GROUP, - ), - ] - return res - - -class DumpDescriptionOptions(Options): - def __init__(self): - self.dump_all_conf_docs = False - self.conf_docs_json = False - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--dump-all'], - help='Dump information for all entities including internal', - hook=SetConstValueHook('dump_all_conf_docs', True), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--json'], - help='Dump information for all entities including internal in json format, uses are included', - hook=SetConstValueHook('conf_docs_json', True), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DumpAtdRevisionOptions(Options): - def __init__(self): - self.human_readable = False - self.print_size = False - self.arcadia_revision = None - self.path = None - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--human-readable'], - help='Add human-readable paths to comments', - hook=SetConstValueHook('human_readable', True), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--print-size'], - help='Add human-readable paths to comments', - hook=SetConstValueHook('print_size', True), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--revision'], - help='Gather revisions for this arcadia revision', - hook=SetValueHook('arcadia_revision'), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--path'], - help='Gather revisions for this path recursively', - hook=SetValueHook('path'), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DumpModuleInfoOptions(Options): - def __init__(self): - self.modules_info_file = '' - self.modules_info_filter = '' - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--modules-info-file'], - help='Save modules informaton into file specified', - hook=SetValueHook('modules_info_file'), - group=DUMP_OPTS_GROUP, - ), - ArgConsumer( - ['--modules-info-filter'], - help='Dump information only for modules matching regexp', - hook=SetValueHook('modules_info_filter'), - group=DUMP_OPTS_GROUP, - ), - ] - - -class DumpRecipesOptions(Options): - def __init__(self): - self.json_format = False - - def consumer(self): - res = [ - ArgConsumer( - ['--json'], - help='Dump recipes in json format', - hook=SetConstValueHook('json_format', True), - group=DUMP_OPTS_GROUP, - ), - ] - return res - - -def do_dump_srcdeps(params): - debug_options = [] - params.flags['YMAKE_ADD_DATA'] = 'yes' - if params.with_yamake: - debug_options.append('mkf') - _do_dump(gen_srcdeps, params, debug_options) - - -def do_dot_graph(params): - _do_dump(gen_modules, params, ['D']) - - -def do_dep_graph(params): - options = params.legacy_deps_opts - if params.output_type == DepGraphOutputType.FLAT_JSON: - options.append('flat-json-with-cmds') - elif params.output_type == DepGraphOutputType.FLAT_JSON_FILES: - options.append('flat-json') - _do_dump(gen_graph, params, options) - - -def do_json_dep_graph(params): - _do_dump(gen_json_graph, params, params.legacy_deps_opts) - - -def do_licenses(params): - _do_dump( - gen_licenses_list, - params, - write_stdout=True, - debug_options=['lic-json' if params.json_licenses else 'lic'], - lic_link_type=params.link_type, - lic_custom_tags=params.custom_tags, - ) - - -def do_forced_deps(params, write_stdout=True): - res = _do_dump( - gen_forced_deps_list, - params, - write_stdout=write_stdout, - debug_options=['fdm-json' if params.json_forced_deps else 'fdm'], - ) - return '' if write_stdout else res.stdout - - -# TODO: support host/target platforms opts -def do_build_plan(params): - with temp_dir() as tmp: - graph = gen_plan( - arc_root=params.arc_root, - build_root=tmp, - build_type=params.build_type, - build_targets=params.abs_targets, - debug_options=params.debug_options, - flags=params.flags, - ymake_bin=params.ymake_bin, - no_ymake_resource=params.no_ymake_resource, - vcs_file=params.vcs_file, - ) - json.dump(graph, sys.stdout, indent=4, sort_keys=True) - - -def do_relation(params): - options = [] - if params.recursive: - options.append('recursive') - for dst in params.relation_dst: - _do_dump( - gen_relation, - params, - options, - find_path_from=params.relation_src, - find_path_to=[dst], - ) - - -def do_all_relations(params): - options = ['J' if params.json_format else 'D'] - if params.recursive: - options.append('recursive') - if params.show_targets_deps: - options.append('show-targets-deps') - _do_dump( - gen_relation, - params, - options, - find_path_from=params.relation_src, - find_path_to=params.relation_dst, - ) - - -def do_files(params): - options = [] - if params.with_data: - options = ['dump-data'] - params.flags['YMAKE_ADD_DATA'] = 'yes' - if params.skip_make_files: - options.append('skip-make-files') - if params.mark_make_files: - options.append('mark-make-files') - _do_dump(gen_filelist, params, options) - - -def do_loops(params): - _do_dump(gen_all_loops, params) - - -def do_peerdir_loops(params): - _do_dump(gen_dir_loops, params) - - -def do_imprint(params): - lines = imprint.generate_detailed_imprints(os.curdir) - writer = csv.writer(sys.stdout, delimiter='\t') - for line in lines: - writer.writerow(line) - - -# TODO: support host/target platforms opts -def do_uids(params): - uids = gen_uids( - arc_root=params.arc_root, - build_root=None, - build_type=params.build_type, - build_targets=params.abs_targets, - debug_options=[], - flags={}, - ymake_bin=None, - ) - json.dump(uids, sys.stdout, indent=4, sort_keys=True) - - -def get_tests(params): - params.flags['TRAVERSE_RECURSE_FOR_TESTS'] = 'yes' - kwargs = { - 'build_root': params.bld_root, - 'build_type': params.build_type, - 'abs_targets': params.abs_targets, - 'debug_options': [], - 'flags': params.flags, - 'ymake_bin': params.ymake_bin, - 'arcadia_tests_data_path': 'arcadia_tests_data', - } - ( - _, - test_dart, - ) = gen_test_dart(**kwargs) - - return sorted( - generate_tests_by_dart( - test_dart, - opts=params, - ), - key=lambda test: os.path.join(test.project_path, test.name), - ) - - -def do_json_test_list(params): - json.dump([t.save() for t in get_tests(params)], sys.stdout, indent=4, sort_keys=True) - - -def do_test_list(params): - print(pickle.dumps(get_tests(params))) - - -def do_conf(params): - with temp_dir() as tmp: - generation_conf = _do_dump(gen_conf, params, write_stdout=False, build_root=tmp) - print(open(generation_conf, 'r').read()) - - -def do_conf_docs(params): - _do_dump(dump_mmm_docs, params, dump_all_conf_docs=params.dump_all_conf_docs, conf_docs_json=params.conf_docs_json) - - -def do_dump_raw_vcs_info(params): - sys.stdout.write(json.dumps(vcsversion.get_raw_version_info(params.arc_root, params.bld_root))) - - -def do_dump_vcs_info(params): - fake_data = strtobool(params.flags.get('NO_VCS_DEPENDS', 'no')) - fake_build_info = strtobool(params.flags.get('CONSISTENT_BUILD', 'no')) - sys.stdout.write(vcsversion.get_version_info(params.arc_root, params.bld_root, fake_data, fake_build_info) + '\n') - - -def do_dump_svn_revision(params): - sys.stdout.write(str(vcsversion.repo_config(params.arc_root))) - - -def do_recipes(params): - tests = get_tests(params) - test_recipes = [] - for test in tests: - encoded_recipes = test.recipes - if not encoded_recipes: - continue - decoded_recipes = decode_recipe_cmdline(encoded_recipes) - test_recipes.append(decoded_recipes) - - recipes = set() - for test_recipe in test_recipes: - for recipe in test_recipe: - recipes.add(tuple(recipe)) - - if params.json_format: - output = json.dumps(recipes, indent=4, sort_keys=True) - else: - recipes_str = [' '.join(recipe) for recipe in recipes] - output = ' '.join(recipes_str) - print(output) diff --git a/devtools/ya/handlers/dump/gen_conf_docs.py b/devtools/ya/handlers/dump/gen_conf_docs.py deleted file mode 100644 index 4b9978f2264..00000000000 --- a/devtools/ya/handlers/dump/gen_conf_docs.py +++ /dev/null @@ -1,384 +0,0 @@ -from __future__ import absolute_import, unicode_literals -import exts.yjson as json -import re -import os -import six - -import core.config - -import build.genconf -import build.ymake2 - -from yalibrary.vcs import vcsversion -from build.build_facade import gen_conf -from exts.strtobool import strtobool - - -class _Markdown: - header = '#' - alink = 'https://a.yandex-team.ru/arc/trunk/arcadia/' - # Section nesting count - scount = 2 - # Description nesting count - dcount = 6 - # Avoid too many entries in toc for macros - chunks_in_toc = 10 - # Checks description - internal_pattern = re.compile(r'^\s*@?internal[.]?\s*$', flags=re.IGNORECASE) - # Checks description - deprecated_pattern = re.compile(r'^\s*@?deprecated[.]?\s*$', flags=re.IGNORECASE) - # Checks header - h_patterns = { - 'internal': re.compile('.*#.*internal.*', flags=re.IGNORECASE), - 'deprecated': re.compile('.*#.*deprecated.*', flags=re.IGNORECASE), - } - - def __init__(self, arc_root, dump_all_descs, use_svn): - self.descs = {'macros': {}, 'modules': {}, 'multimodules': {}, 'unknowns': {}} - self.links = {} - self.anchors = {} - try: - self.svn_revision, _ = vcsversion.repo_config(arc_root) if use_svn else (-1, '') - except Exception: - self.svn_revision = -1 - self.dump_all_descs = dump_all_descs - - def process_entry(self, file, entry): - if 'kind' not in entry or entry['kind'] != 'node': - return - - self._add_entry_optionally(file, entry) - - def dump(self): - res = self._dump_toc() - - for type in ['multimodules', 'modules', 'macros', 'unknowns']: - if self.descs[type]: - res += _Markdown._format_section(type) - - for name in sorted(self.descs[type]): - for d in self.descs[type][name]['text']: - res += six.ensure_str(d) - - for type in ['multimodules', 'modules', 'macros', 'unknowns']: - if self.descs[type]: - for name in sorted(self.descs[type]): - link = self.descs[type][name]['link'] - res += six.ensure_str(link) - - return res - - def _add_entry_optionally(self, file, entry): - props = entry['properties'] - doc = {'name': props['name'], 'type': props['type'], 'file': file, 'line': entry['range']['line']} - - if 'comment' in props: - doc['long desc'] = props['comment'] - if 'usage' in props: - doc['usage'] = props['usage'] - - doc['revision'] = self.svn_revision - - descs, link = self._format_entry(doc) - - dictionary = ( - self.descs[doc['type'] + 's'] - if doc['type'] in ['macro', 'module', 'multimodule'] - else self.descs['unknowns'] - ) - - if not self.dump_all_descs and _Markdown._is_internal(descs[0]): - return - - dictionary[doc['name']] = {'text': descs, 'link': link, 'src_data': doc} - - @classmethod - def _is_internal(cls, header): - return cls.h_patterns['internal'].match(header) - - def _dump_toc(self): - res = '*Do not edit, this file is generated from comments to macros definitions using `ya dump conf-docs{all}`.*\n\n'.format( - all=' --dump-all' if self.dump_all_descs else '' - ) - res += '{markup} ya.make {all}commands\n\n'.format( - markup=_Markdown.header, all='and core.conf ' if self.dump_all_descs else '' - ) - res += ( - 'General info: [How to write ya.make files](https://wiki.yandex-team.ru/yatool/HowToWriteYaMakeFiles)\n\n' - ) - res += '{markup} Table of contents\n\n'.format(markup=_Markdown.header * 2) - - for type in ['multimodules', 'modules', 'macros', 'unknowns']: - if self.descs[type]: - res += _Markdown._format_toc_section(type) - if type != 'macros': - for name in sorted(self.descs[type]): - res += _Markdown._format_toc_header(self.descs[type][name]['src_data']) - else: - chunk_cnt = 0 - first_macro = {} - last_macro = {} - for name in sorted(self.descs[type]): - if chunk_cnt == 0: - first_macro = self.descs[type][name]['src_data'] - chunk_cnt += 1 - last_macro = self.descs[type][name]['src_data'] - - if chunk_cnt == _Markdown.chunks_in_toc: - chunk_cnt = 0 - res += _Markdown._format_toc_macro_header(first_macro, last_macro) - - if chunk_cnt != 0: - res += _Markdown._format_toc_macro_header(first_macro, last_macro) - return res - - @classmethod - def _format_entry(cls, doc): - descs = [] - - lines, special_tags = _Markdown._format_desc(doc) - descs.append(_Markdown._format_header(doc, special_tags)) - descs.append(lines) - return (descs, _Markdown._format_link(doc)) - - @classmethod - def _format_toc_section(cls, type): - return '{indent} * [{section}](#{anchor})\n'.format( - indent=' ' * cls.scount, section=type.capitalize(), anchor=type - ) - - @classmethod - def _format_section(cls, type): - return '{markup} {text} <a name="{anchor}"></a>\n\n'.format( - markup=cls.header * cls.scount, text=type.capitalize(), anchor=type - ) - - @staticmethod - def _format_header_anchor(doc): - return '<a name="{atype}_{aname}"></a>'.format(atype=doc['type'], aname=doc['name']) - - @classmethod - def _format_toc_header(cls, doc): - qual = doc['type'].capitalize() if doc['type'] in ['macro', 'module', 'multimodule'] else "Unknown" - return '{indent} - {qual} [{name}](#{type}_{name})\n'.format( - indent=' ' * cls.dcount, qual=qual, name=doc['name'], type=doc['type'] - ) - - @classmethod - def _format_toc_macro_header(cls, fdoc, ldoc): - return '{indent} - Macros [{fname}](#{ftype}_{fname}) .. [{lname}](#{ltype}_{lname})\n'.format( - indent=' ' * cls.dcount, fname=fdoc['name'], ftype=fdoc['type'], lname=ldoc['name'], ltype=ldoc['type'] - ) - - # Also adds special tags 'internal' and 'deprecated' - @classmethod - def _format_header(cls, doc, special_tags): - name = doc['name'] - usage = doc['usage'] if 'usage' in doc else "" - usage = name if not usage else usage - - qual = doc['type'].capitalize() if doc['type'] in ['macro', 'module', 'multimodule'] else "Unknown" - - usage = _Markdown._remove_formatting(usage.rstrip().lstrip()) - name = _Markdown._remove_formatting(name) - - usage = usage.replace(name, '[' + name + "][]", 1) - - if special_tags: - special = '' - for tag in special_tags: - if not cls.h_patterns[tag].match(usage): - special += ' ' + tag - # Emphasis - if usage.find("#") != -1: - usage += special - else: - usage += ' #' + special[1:] - - # Emphasis - if usage.find("#") != -1: - usage = usage.replace('#', "_#", 1) - usage += "_" - - return '{markup} {type} {usage} {anchor}\n'.format( - markup=cls.header * cls.dcount, type=qual, usage=usage, anchor=_Markdown._format_header_anchor(doc) - ) - - # Prints verbatim. Strips unnecessary indent and escapes '_'/'*'. - @classmethod - def _format_desc(cls, doc): - result = "" - - desc = "" - if 'long desc' in doc: - desc = doc['long desc'].rstrip() - if not desc: - desc = " Not documented yet.\n" - - lines = _Markdown._strip_blanks(desc.splitlines()) - lines = _Markdown._remove_formatting_markdown(lines) - - result += '\n'.join(lines) + "\n\n" - - special_tags = [] - if doc['name'].startswith('_') or any([cls.internal_pattern.match(x) for x in lines]): - special_tags.append("internal") - if any([cls.deprecated_pattern.match(x) for x in lines]): - special_tags.append("deprecated") - - return (result, special_tags) - - @staticmethod - def _format_link(doc): - return ' [{tag_name}]: {baselink}{file}{rev}#L{line}\n'.format( - tag_name=_Markdown._remove_formatting(doc['name']), - baselink=_Markdown.alink, - file=doc['file'], - rev='?rev=' + str(doc['revision']) if doc['revision'] > 0 else '', - line=doc['line'], - ) - - @staticmethod - def _strip_blanks(lines): - first = 0 - for line in lines: - if not line.lstrip().rstrip(): - first += 1 - else: - break - last = 0 - for line in reversed(lines): - if not line.lstrip().rstrip(): - last += 1 - else: - break - lines = lines[first : (len(lines) - last)] - lines = [x.rstrip().expandtabs(4) for x in lines] - indent = 10000 - for line in lines: - if line: - indent = min(indent, len(line) - len(line.lstrip())) - - if indent > 0: - lines = [x.replace(' ' * indent, '', 1) for x in lines] - - return lines - - # Conditionally removed formatting. - # Code blocks are not modified - @staticmethod - def _remove_formatting_markdown(lines): - code = False - new_paragraph = True - backtick_block = False - - res = [] - for line in lines: - if new_paragraph: - if line.startswith('\t') or line.startswith(' ' * 4): - code = True - - if not line.startswith('\t') and not line.startswith(' ' * 4): - code = False - - new_paragraph = not line - - if line.lstrip() == '```': - backtick_block = not backtick_block - - res.append(line if code or backtick_block else _Markdown._remove_formatting(line)) - return res - - # Unconditionally removes formatting due to '_'/'*' - @staticmethod - def _remove_formatting(x): - return x.replace("_", r"\_").replace("*", r"\*") - - -def _gen( - custom_build_directory, - build_type, - build_targets, - debug_options, - flags=None, - warn_mode=None, - ymake_bin=None, - platform=None, - host_platform=None, - target_platforms=None, - **kwargs -): - generation_conf = gen_conf( - build_root=custom_build_directory, - build_type=build_type, - build_targets=build_targets, - flags=flags, - host_platform=host_platform, - target_platforms=target_platforms, - ) - res, evlog_dump = build.ymake2.ymake_dump( - custom_build_directory=custom_build_directory, - build_type=build_type, - abs_targets=build_targets, - debug_options=debug_options, - warn_mode=warn_mode, - flags=flags, - ymake_bin=ymake_bin, - platform=platform, - grab_stderr=True, - custom_conf=generation_conf, - **kwargs - ) - return res - - -def dump_mmm_docs( - build_root, - build_type, - build_targets, - debug_options, - flags, - dump_all_conf_docs=None, - conf_docs_json=None, - ymake_bin=None, - platform=None, - host_platform=None, - target_platforms=None, -): - json_dump_name = os.path.join(build_root, 'ymake.dump.ydx.json') - arc_root = core.config.find_root_from(build_targets) - null_ya_make = os.path.join(arc_root, 'build', 'docs', 'empty') - - if not conf_docs_json: - if not os.path.exists(null_ya_make): - raise "Empty project not found, dump conf-docs may work too long" - - res = _gen( - custom_build_directory=build_root, - build_type=build_type, - # Override target - build_targets=[null_ya_make] if not conf_docs_json else build_targets, - debug_options=debug_options, - flags=flags, - ymake_bin=ymake_bin, - platform=platform, - host_platform=host_platform, - target_platforms=target_platforms, - yndex_file=json_dump_name, - ) - - if conf_docs_json: - with open(json_dump_name, 'r') as jfile: - res.stdout += jfile.read() - else: - with open(json_dump_name, 'r') as jfile: - contents = jfile.read() - jdata = json.loads(contents) - no_svn = True if 'NO_SVN_DEPENDS' in flags and strtobool(flags['NO_SVN_DEPENDS']) else False - doc = _Markdown(arc_root, dump_all_conf_docs, not no_svn) - for efile in jdata: - for entry in jdata[efile]: - doc.process_entry(efile, entry) - res.stdout += doc.dump() - - return res diff --git a/devtools/ya/handlers/dump/ya.make b/devtools/ya/handlers/dump/ya.make deleted file mode 100644 index adbc66ad9fa..00000000000 --- a/devtools/ya/handlers/dump/ya.make +++ /dev/null @@ -1,52 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.dump - __init__.py - gen_conf_docs.py -) - -PEERDIR( - contrib/python/pathlib2 - contrib/python/six - contrib/python/toml - devtools/ya/app - devtools/ya/build - devtools/ya/build/build_facade - devtools/ya/build/build_opts - devtools/ya/build/genconf - devtools/ya/build/ymake2 - devtools/ya/core - devtools/ya/core/config - devtools/ya/core/imprint - devtools/ya/core/yarg - devtools/ya/exts - devtools/ya/test/dartfile - devtools/ya/test/explore - devtools/ya/yalibrary/debug_store - devtools/ya/yalibrary/debug_store/processor - devtools/ya/yalibrary/debug_store/store - devtools/ya/yalibrary/tools - devtools/ya/yalibrary/vcs - devtools/ya/yalibrary/vcs/vcsversion - devtools/ya/yalibrary/yandex/sandbox/misc - library/python/fs - library/python/func - library/python/tmp -) - -IF (NOT YA_OPENSOURCE) - PEERDIR( - devtools/ya/handlers/dump/arcadia_specific - devtools/ya/handlers/dump/debug - ) -ENDIF() - -END() - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/gc/__init__.py b/devtools/ya/handlers/gc/__init__.py deleted file mode 100644 index 824a35edf29..00000000000 --- a/devtools/ya/handlers/gc/__init__.py +++ /dev/null @@ -1,342 +0,0 @@ -from __future__ import absolute_import -import glob -import os -import logging -import sys -import time -import six - -import app -import core.yarg -import core.common_opts -import core.config as cc -import build.ya_make as ym -from exts import fs -from core.common_opts import CustomBuildRootOptions -from build.build_opts import LocalCacheOptions, DistCacheSetupOptions, parse_size_arg, parse_timespan_arg -from exts.windows import on_win -from yalibrary.runner import result_store -import yalibrary.toolscache as tc - -if six.PY3: - long = int - -logger = logging.getLogger(__name__) - - -def _to_size_in_gb(size): - try: - return int(float(size) * 1024 * 1024 * 1024) - except ValueError: - pass - - return parse_size_arg(size) - - -def _to_size_in_mb(size): - try: - return int(float(size) * 1024 * 1024) - except ValueError: - pass - - return parse_size_arg(size) - - -def _to_timespan_in_hours(timespan): - try: - return int(float(timespan) * 60 * 60) - except ValueError: - pass - - return parse_timespan_arg(timespan) - - -class CollectCacheOptions(LocalCacheOptions): - def __init__(self): - super(CollectCacheOptions, self).__init__() - self.object_size_limit = None - self.age_limit = None - self.symlinks_ttl = 0 - - def consumer(self): - return super(CollectCacheOptions, self).consumer() + [ - core.yarg.ArgConsumer( - ['--size-limit'], - help='Strip build cache to size (in GiB if not set explicitly)', - hook=core.yarg.SetValueHook( - 'cache_size', - transform=_to_size_in_gb, - default_value=lambda x: str(_to_size_in_gb(x) * 1.0 / 1024 / 1024 / 1024), - ), - group=core.yarg.BULLET_PROOF_OPT_GROUP, - visible=False, - ), - core.yarg.ArgConsumer( - ['--object-size-limit'], - help='Strip build cache from large objects (in MiB if not set explicitly)', - hook=core.yarg.SetValueHook('object_size_limit', transform=_to_size_in_mb), - group=core.yarg.BULLET_PROOF_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--age-limit'], - help='Strip build cache from old objects (in hours if not set explicitly)', - hook=core.yarg.SetValueHook('age_limit', transform=_to_timespan_in_hours), - group=core.yarg.BULLET_PROOF_OPT_GROUP, - ), - ] - - -class GarbageCollectionYaHandler(core.yarg.CompositeHandler): - def __init__(self): - core.yarg.CompositeHandler.__init__(self, description='Collect garbage') - self['cache'] = core.yarg.OptsHandler( - action=app.execute(action=do_cache, respawn=app.RespawnType.OPTIONAL), - description='Strip build cache and old build directories', - opts=[core.common_opts.ShowHelpOptions(), CollectCacheOptions(), CustomBuildRootOptions()], - visible=True, - ) - self['dist_cache'] = core.yarg.OptsHandler( - action=app.execute(action=do_strip_yt_cache, respawn=app.RespawnType.NONE), - description='Strip distributed (YT) cache', - opts=[core.common_opts.ShowHelpOptions(), DistCacheSetupOptions()], - visible=False, - ) - - -class FilterBySize(object): - def __init__(self, size_limit): - self.size_limit = size_limit - self.total_size = 0 - self._items = {} - - def __call__(self, item): - if item.uid in self._items: - return False - size = item.size - self.total_size += size - self._items[item.uid] = item - return self.total_size < self.size_limit - - -class FilterByObjectSize(object): - def __init__(self, size_limit): - self.size_limit = size_limit - - def __call__(self, item): - return item.size < self.size_limit - - -class FilterByAge(object): - def __init__(self, age_limit): - self.age_limit = age_limit - self.now = time.time() - - def __call__(self, item): - return item.timestamp > self.now - self.age_limit - - -def _clean_dir_selectively(logs, file_dir, file_name): - try: - for i in os.listdir(logs): - date_dir = os.path.join(logs, i) - if date_dir != file_dir: - fs.remove_tree_safe(date_dir) - - if file_dir: - for i in os.listdir(file_dir): - if i != file_name: - fs.remove_tree_safe(os.path.join(file_dir, i)) - except OSError as e: - logger.debug('Cleaning %s root failed %s', logs, e) - - -def _clean_logs(): - logger.debug('Cleaning logs root') - - try: - import app_ctx - - file_dir, file_name = os.path.split(app_ctx.file_log) - except Exception as e: - file_dir, file_name = ('', '') - logger.debug('Log name was not obtained %s', e) - - logs = os.path.join(cc.misc_root(), 'logs') - _clean_dir_selectively(logs, file_dir, file_name) - - -def _clean_evlogs(): - logger.debug('Cleaning evlogs root') - - try: - import app_ctx - - file_dir, file_name = os.path.split(app_ctx.evlog._fileobj.name) - except Exception as e: - file_dir, file_name = ('', '') - logger.debug('Log name was not obtained %s', e) - - evlogs = os.path.join(cc.misc_root(), 'evlogs') - _clean_dir_selectively(evlogs, file_dir, file_name) - - -def _clean_tools(): - running_ya_bin_dir = os.path.dirname(sys.argv[0]) - base_running_ya_bin_dir = os.path.basename(running_ya_bin_dir).replace('_d', '') - errors = 0 - for i in os.listdir(os.path.dirname(cc.tool_root())): - full_path = os.path.join(os.path.dirname(cc.tool_root()), i) - if i == 'v4': - if len(glob.glob(os.path.join(full_path, '*.corrupted'))) == 0: - tc.unlock_all() - continue - else: - errors += 1 - elif full_path == running_ya_bin_dir: - continue - elif os.path.isfile(full_path): - try: - if os.path.getsize(full_path) < 1024 * 100: - with open(full_path, 'r') as f: - if base_running_ya_bin_dir in f.read(): - continue - except Exception: - errors += 1 - logger.debug('Cleaning tool %s', full_path) - fs.remove_tree_safe(full_path) - return errors - - -def do_cache(opts): - build_root = opts.custom_build_directory or cc.build_root() - - cache = None - try: - cache = ym.make_cache(opts, build_root) - except Exception: - logger.exception("While initializing cache") - - lock = ym.make_lock(opts, build_root, write_lock=True) - - with lock: - if opts.cache_stat: - import app_ctx - - if not cache: - app_ctx.display.emit_message("Cache not initialized, can't show stats") - else: - cache.analyze(app_ctx.display) - errors = 0 - else: - errors = _do_collect_cache(cache, build_root, opts) - - if cache: - cache.flush() - - logger.debug( - "ya gc stats %s", - { - 'cache_size': opts.cache_size, - 'object_size_limit': opts.object_size_limit, - 'age_limit': opts.age_limit, - 'symlinks_ttl': opts.symlinks_ttl, - 'errors': errors, - }, - ) - - -def _do_collect_cache(cache, build_root, opts): - logger.debug('Cleaning tmp root') - fs.remove_tree_safe(cc.tmp_path()) - - logger.debug('Cleaning snowden root') - fs.remove_tree_safe(os.path.join(cc.misc_root(), 'snowden')) - - logger.debug('Cleaning build root') - fs.remove_tree_safe(os.path.join(build_root, 'build_root')) - - logger.debug('Cleaning conf root') - fs.remove_tree_safe(os.path.join(build_root, 'conf')) - - errors = _clean_tools() - - logger.debug('Cleaning tmp root') - fs.remove_tree_safe(os.path.join(cc.misc_root(), 'tmp')) - - _clean_logs() - _clean_evlogs() - - if not on_win(): - logger.debug('Cleaning symres') - symres_dir = os.path.join(build_root, 'symres') - src_dir = cc.find_root(fail_on_error=False) or "" - if os.path.isdir(symres_dir): - logger.debug('Cleaning symres %s for %s, ttl=%s', symres_dir, src_dir, opts.symlinks_ttl) - result_store.SymlinkResultStore(symres_dir, src_dir).sieve( - state=None, ttl=opts.symlinks_ttl, cleanup=opts.symlinks_ttl == 0 - ) - - if hasattr(cache, '_store_path'): - for dir in os.listdir(os.path.join(build_root, 'cache')): - full_path = os.path.join(build_root, 'cache', dir) - if cache._store_path == full_path: - if len(glob.glob(os.path.join(full_path, '*.corrupted'))) == 0: - continue - else: - errors += 1 - logger.debug('Cleaning cache directory %s', full_path) - fs.remove_tree_safe(full_path) - - if opts.cache_size is not None and opts.object_size_limit is None and opts.age_limit is None: - logger.debug('Cleaning for total size %s', opts.cache_size) - if hasattr(cache, 'strip'): - cache.strip(FilterBySize(opts.cache_size)) - elif hasattr(cache, 'strip_total_size'): - cache.strip_total_size(opts.cache_size) - - from yalibrary.toolscache import tc_force_gc - - tc_force_gc(opts.cache_size) - - if opts.object_size_limit is not None: - logger.debug('Cleaning for object size %s', opts.object_size_limit) - if hasattr(cache, 'strip'): - cache.strip(FilterByObjectSize(opts.object_size_limit)) - elif hasattr(cache, 'strip_max_object_size'): - cache.strip_max_object_size(opts.object_size_limit) - - if opts.age_limit is not None: - logger.debug('Cleaning for age %s', opts.age_limit) - if hasattr(cache, 'strip'): - cache.strip(FilterByAge(opts.age_limit)) - elif hasattr(cache, 'strip_max_age'): - cache.strip_max_age(opts.age_limit) - - return errors - - -def do_strip_yt_cache(opts): - try: - from yalibrary.store.yt_store import yt_store - except ImportError as e: - logger.warn("YT store is not available: %s", e) - - token = opts.yt_token or opts.oauth_token - cache = yt_store.YtStore( - opts.yt_proxy, - opts.yt_dir, - None, - token=token, - readonly=opts.yt_readonly, - max_cache_size=opts.yt_max_cache_size, - ttl=opts.yt_store_ttl, - ) - - counters = cache.strip() - if counters: - logger.info( - 'Deleted: meta rows:%d, data rows:%d, net data size:%d', - counters['meta_rows'], - counters['data_rows'], - counters['data_size'], - ) diff --git a/devtools/ya/handlers/gc/ya.make b/devtools/ya/handlers/gc/ya.make deleted file mode 100644 index 1b8296d1ee1..00000000000 --- a/devtools/ya/handlers/gc/ya.make +++ /dev/null @@ -1,37 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.gc - __init__.py -) - -PEERDIR( - contrib/python/six - devtools/ya/app - devtools/ya/build - devtools/ya/build/build_opts - devtools/ya/core - devtools/ya/core/config - devtools/ya/core/yarg - devtools/ya/exts - devtools/ya/yalibrary/runner - devtools/ya/yalibrary/store - devtools/ya/yalibrary/toolscache - library/python/fs - library/python/windows -) - -IF (NOT YA_OPENSOURCE) - PEERDIR( - devtools/ya/yalibrary/store/yt_store # see YA-938 - ) -ENDIF() - -END() - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/gen_config/__init__.py b/devtools/ya/handlers/gen_config/__init__.py deleted file mode 100644 index e771dac8715..00000000000 --- a/devtools/ya/handlers/gen_config/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import absolute_import -import core.common_opts -import core.yarg - -from . import gen_config - - -class GenConfigOptions(core.yarg.Options): - def __init__(self): - self.output = None - self.dump_defaults = False - - @staticmethod - def consumer(): - return [ - core.yarg.SingleFreeArgConsumer( - help='ya.conf', - hook=core.yarg.SetValueHook('output'), - required=False, - ), - core.yarg.ArgConsumer( - ['--dump-defaults'], - help='Dump default values as JSON', - hook=core.yarg.SetConstValueHook('dump_defaults', True), - ), - ] - - -class GenConfigYaHandler(core.yarg.OptsHandler): - description = 'Generate default ya config' - - def __init__(self): - self._root_handler = None - super(GenConfigYaHandler, self).__init__( - action=self.do_generate, - description=self.description, - opts=[ - core.common_opts.ShowHelpOptions(), - GenConfigOptions(), - ], - ) - - def handle(self, root_handler, args, prefix): - self._root_handler = root_handler - super(GenConfigYaHandler, self).handle(root_handler, args, prefix) - - def do_generate(self, args): - gen_config.generate_config(self._root_handler, args.output, args.dump_defaults) diff --git a/devtools/ya/handlers/gen_config/gen_config.py b/devtools/ya/handlers/gen_config/gen_config.py deleted file mode 100644 index 56a0d6fc64d..00000000000 --- a/devtools/ya/handlers/gen_config/gen_config.py +++ /dev/null @@ -1,282 +0,0 @@ -from __future__ import absolute_import -import sys -import itertools -import collections -import json - -import toml - -import core.yarg -import six - -toml_encoder = toml.TomlEncoder() - -NOT_SET = -1 - - -def is_jsonable(x): - try: - json.dumps(x) - return True - except (TypeError, OverflowError): - return False - - -def iter_ya_options(root_handler, all_opts=False): - # prefer setvalue over setconstvalue to provide correct description and default value - hook_prefer_order = [core.yarg.SetValueHook, core.yarg.SetConstValueHook] - - def get_hook_priority(hook, consumer): - if isinstance(consumer, core.yarg.ConfigConsumer): - return -1 - - htype = type(hook) - if htype in hook_prefer_order: - pos = hook_prefer_order.index(htype) - else: - pos = len(hook_prefer_order) - return len(hook_prefer_order) - pos - - for handler_name, command in six.iteritems(root_handler.sub_handlers): - if not command.visible: - continue - - top_handler = command.command() - handlers = collections.deque([([handler_name], top_handler)]) - - while handlers: - trace, handler = handlers.pop() - - sub_handlers = handler.sub_handlers or {} - for sub_name in sorted(sub_handlers): - handlers.append((trace + [sub_name], sub_handlers[sub_name])) - - options = getattr(handler, "options", None) - if not options: - continue - - if all_opts: - for opt in options: - for name in vars(opt).keys(): - default = getattr(opt, name) - if is_jsonable(default): - yield name, "", default, "", "", "" - continue - - top_consumer = options.consumer() - if isinstance(top_consumer, core.yarg.Compound): - consumers = top_consumer.parts - else: - consumers = [top_consumer] - - # pairwise config option with command line name if it's possible - params = {} - for consumer in consumers: - if not isinstance(consumer, (core.yarg.ArgConsumer, core.yarg.ConfigConsumer)): - continue - - hook = consumer.hook - if not hook: - continue - - opt_name = getattr(hook, 'name', None) - if not opt_name or opt_name.startswith("_"): - continue - - if opt_name not in params: - params[opt_name] = { - 'configurable': False, - } - - entry = params[opt_name] - priority = get_hook_priority(hook, consumer) - - entry['configurable'] |= isinstance(consumer, core.yarg.ConfigConsumer) - if priority > entry.get('_priority', -2): - entry['cmdline_names'] = get_arg_consumer_names(consumer) or entry.get('cmdline_names') - entry['trace'] = trace or entry.get('trace') - entry['description'] = getattr(consumer, "help", "") or entry.get('description') - entry['group'] = getattr(consumer, "group", "") or entry.get('group', 0) - # default value might be empty list or dict - if getattr(options, opt_name, None) is None: - entry['default'] = entry.get('default') - else: - entry['default'] = getattr(options, opt_name) - entry['_priority'] = priority - - for name, x in params.items(): - if x['configurable']: - yield name, x['cmdline_names'], x['default'], x['group'], x['description'], x['trace'] - - -def get_arg_consumer_names(consumer): - if isinstance(consumer, core.yarg.ArgConsumer): - return list([_f for _f in [consumer.short_name, consumer.long_name] if _f]) - return [] - - -def split(iterable, func): - d1 = [] - d2 = [] - for entry in iterable: - if func(entry): - d1.append(entry) - else: - d2.append(entry) - return d1, d2 - - -def get_comment(entry): - parts = [] - if entry['desc']: - parts.append(entry['desc']) - if entry['cmdline_names']: - parts.append("({})".format(", ".join(entry['cmdline_names']))) - return " ".join(parts) - - -def compress_keys(data, result, trace=None): - trace = trace or [] - for key, val in six.iteritems(data): - if isinstance(val, dict): - result[".".join(trace + [key])] = val - compress_keys(val, result, trace + [key]) - - -def dump_options(entries, output, config, section="", print_comment=True): - if section: - output.append("[{}]".format(section)) - for entry in sorted(entries, key=lambda x: x['name']): - parts = [] - if print_comment: - comment = get_comment(entry) - if comment: - parts.append("# {}".format(comment)) - name = entry['name'] - if name in config and config[name] != entry['default']: - parts.append("{} = {}".format(name, toml_encoder.dump_value(config[name]))) - else: - parts.append("# {} = {}".format(name, toml_encoder.dump_value(entry['default']))) - - output.append("\n".join(parts)) - - -def dump_subgroups(subgroups, output, config, section="", print_comment=True): - def get_section(name): - return ".".join([_f for _f in [section, name] if _f]) - - for entry in subgroups: - parts = [] - if print_comment: - comment = get_comment(entry) - if comment: - parts.append("# {}".format(comment)) - - name = entry['name'] - if name in config and config[name] != entry['default']: - parts.append("[{}]".format(get_section(name))) - for key, val in config[name].items(): - parts.append("{} = {}".format(key, toml_encoder.dump_value(val))) - else: - parts.append("# [{}]".format(get_section(name))) - for key, val in entry['default'].items(): - parts.append("# {} = {}".format(key, toml_encoder.dump_value(val))) - - output.append("\n".join(parts)) - - -def dump_config(options, handler_map, user_config): - def sort_func(x): - return getattr(x['group'], 'index', NOT_SET) - - def get_group_title(name): - return "{} {} {}".format("=" * 10, name, "=" * (65 - len(name))) - - blocks = [ - "# Save config to the junk/{USER}/ya.conf or to the ~/.ya/ya.conf\n# For more info see https://docs.yandex-team.ru/yatool/commands/gen_config" - ] - - # dump all options - subgroups = [] - data = sorted(options.values(), key=sort_func) - for group_index, entries in itertools.groupby(data, sort_func): - entries = list(entries) - - if group_index != NOT_SET: - group_name = entries[0]['group'].name - if group_name: - blocks.append("# " + get_group_title(group_name)) - - subs, opts = split(entries, lambda x: isinstance(x['default'], dict)) - dump_options(opts, blocks, user_config) - # move all subgroup options out of the default section - # otherwise every option defined after section will be related to the section - subgroups += subs - - if subgroups: - blocks.append("# " + get_group_title("Various table options")) - blocks.append("# Uncomment table name with parameters") - dump_subgroups(subgroups, blocks, user_config) - - # Save user redefined opts for specific handlers - specific_opts = {} - user_handler_map = {} - compress_keys(user_config, user_handler_map) - for section, keys in six.iteritems(user_handler_map): - if section not in handler_map: - continue - entries = [] - for optname in keys: - if optname in handler_map[section]: - entries.append(options[optname]) - if entries: - specific_opts[section] = entries - - if specific_opts: - blocks.append("# " + get_group_title("Redefined options for specific handlers")) - for section, entries in sorted(specific_opts.items(), key=lambda x: x[0]): - subs, opts = split(entries, lambda x: isinstance(x['default'], dict)) - if opts: - dump_options(opts, blocks, user_handler_map[section], section, print_comment=False) - if subgroups: - dump_subgroups(subs, blocks, user_handler_map[section], section, print_comment=False) - - return "\n#\n".join(blocks) - - -def generate_config(root_handler, output=None, dump_defaults=None): - # Don't load global config files to avoid penetration of the global options into user config - config_files = core.yarg.get_config_files(global_config=False) - user_config = core.yarg.load_config(config_files) - - options = {} - handler_map = {} - for name, cmdline_names, default, group, desc, trace in iter_ya_options(root_handler, dump_defaults): - if name not in options: - options[name] = { - 'cmdline_names': cmdline_names, - 'default': default, - 'desc': desc, - 'group': group, - 'name': name, - } - else: - entry = options[name] - entry['desc'] = entry['desc'] or desc - entry['group'] = entry['group'] or group - - target = ".".join(trace) - if target not in handler_map: - handler_map[target] = [] - handler_map[target].append(name) - - if dump_defaults: - json.dump({k: v['default'] for k, v in options.items()}, sys.stdout, indent=2) - return - - data = dump_config(options, handler_map, user_config) - if output: - with open(output, 'w') as afile: - afile.write(data) - else: - sys.stdout.write(data + '\n') diff --git a/devtools/ya/handlers/gen_config/ya.make b/devtools/ya/handlers/gen_config/ya.make deleted file mode 100644 index b5fdeece7c7..00000000000 --- a/devtools/ya/handlers/gen_config/ya.make +++ /dev/null @@ -1,23 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.gen_config - __init__.py - gen_config.py -) - -PEERDIR( - contrib/python/six - contrib/python/toml - devtools/ya/core - devtools/ya/core/yarg -) - -END() - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/ide/__init__.py b/devtools/ya/handlers/ide/__init__.py deleted file mode 100644 index 4d28d29a69a..00000000000 --- a/devtools/ya/handlers/ide/__init__.py +++ /dev/null @@ -1,734 +0,0 @@ -from __future__ import absolute_import -import os.path - -import six - -import core.yarg -import core.config -import core.common_opts - -import build.build_opts -import build.compilation_database as bcd - -import ide.ide_common -import ide.msvs -import ide.msvs_lite -import ide.clion2016 -import ide.idea -import ide.qt -import ide.remote_ide_qt -import ide.goland -import ide.pycharm -import ide.venv -import ide.vscode_all -import ide.vscode_clangd -import ide.vscode_go -import ide.vscode_py -import ide.vscode_ts -import ide.vscode.opts - -import yalibrary.platform_matcher as pm - -import app -import app_config - -if app_config.in_house: - import devtools.ya.ide.fsnotifier - -from core.yarg.help_level import HelpLevel - -if six.PY3: - import ide.gradle - - -class TidyOptions(core.yarg.Options): - def __init__(self): - self.setup_tidy = False - - @staticmethod - def consumer(): - return [ - core.yarg.ArgConsumer( - ['--setup-tidy'], - help="Setup default arcadia's clang-tidy config in a project", - hook=core.yarg.SetConstValueHook('setup_tidy', True), - group=core.yarg.BULLET_PROOF_OPT_GROUP, - ), - core.yarg.ConfigConsumer('setup_tidy'), - ] - - -class CLionOptions(core.yarg.Options): - CLION_OPT_GROUP = core.yarg.Group('CLion project options', 0) - - def __init__(self): - self.filters = [] - self.lite_mode = False - self.remote_toolchain = None - self.remote_deploy_config = None - self.remote_repo_path = None - self.remote_build_path = None - self.remote_deploy_host = None - self.use_sync_server = False - self.content_root = None - self.strip_non_final_targets = False - self.full_targets = False - self.add_py_targets = False - - @staticmethod - def consumer(): - return [ - core.yarg.ArgConsumer( - ['--filter', '-f'], - help='Only consider filtered content', - hook=core.yarg.SetAppendHook('filters'), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--mini', '-m'], - help='Lite mode for solution (fast open, without build)', - hook=core.yarg.SetConstValueHook('lite_mode', True), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--remote-toolchain'], - help='Generate configurations for remote toolchain with this name', - hook=core.yarg.SetValueHook('remote_toolchain'), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--remote-deploy-config'], - help='Name of the remote server configuration tied to the remote toolchain', - hook=core.yarg.SetValueHook('remote_deploy_config'), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--remote-repo-path'], - help='Path to the arc repository at the remote host', - hook=core.yarg.SetValueHook('remote_repo_path'), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--remote-build-path'], - help='Path to the directory for CMake output at the remote host', - hook=core.yarg.SetValueHook('remote_build_path'), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--remote-host'], - help='Hostname associated with remote server configuration', - hook=core.yarg.SetValueHook('remote_deploy_host'), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--use-sync-server'], - help='Deploy local files via sync server instead of file watchers', - hook=core.yarg.SetConstValueHook('use_sync_server', True), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--project-root', '-r'], - help='Root directory for a CLion project', - hook=core.yarg.SetValueHook('content_root'), - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--strip-non-final-targets'], - hook=core.yarg.SetConstValueHook('strip_non_final_targets', True), - help='Do not create target for non-final nodes', - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--full-targets'], - hook=core.yarg.SetConstValueHook('full_targets', True), - help='Old Mode: Enable full targets graph generation for project.', - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--add-py3-targets'], - hook=core.yarg.SetConstValueHook('add_py_targets', True), - help='Add Python 3 targets to project', - group=CLionOptions.CLION_OPT_GROUP, - ), - core.yarg.ConfigConsumer('filters'), - core.yarg.ConfigConsumer('lite_mode'), - core.yarg.ConfigConsumer('remote_toolchain'), - core.yarg.ConfigConsumer('remote_deploy_config'), - core.yarg.ConfigConsumer('remote_repo_path'), - core.yarg.ConfigConsumer('remote_build_path'), - core.yarg.ConfigConsumer('remote_deploy_host'), - core.yarg.ConfigConsumer('use_sync_server'), - core.yarg.ConfigConsumer('content_root'), - core.yarg.ConfigConsumer('strip_non_executable_target'), - core.yarg.ConfigConsumer('full_targets'), - core.yarg.ConfigConsumer('add_py_targets'), - ] - - def postprocess2(self, params): - if ' ' in params.project_title: - raise core.yarg.ArgsValidatingException('Clion project title should not contain space symbol') - if params.add_py_targets and params.full_targets: - raise core.yarg.ArgsValidatingException('--add-py-targets must be used without --full-targets') - - -class IdeaOptions(core.yarg.Options): - IDEA_OPT_GROUP = core.yarg.Group('Idea project options', 0) - IDE_PLUGIN_INTEGRATION_GROUP = core.yarg.Group('Integration wuth IDE plugin', 1) - - def __init__(self): - self.idea_project_root = None - self.local = False - self.group_modules = None - self.dry_run = False - self.ymake_bin = None - self.iml_in_project_root = False - self.iml_keep_relative_paths = False - self.idea_files_root = None - self.project_name = None - self.minimal = False - self.directory_based = True - self.omit_test_data = False - self.with_content_root_modules = False - self.external_content_root_modules = [] - self.generate_tests_run = False - self.generate_tests_for_deps = False - self.separate_tests_modules = False - self.auto_exclude_symlinks = False - self.exclude_dirs = [] - self.with_common_jvm_args_in_junit_template = False - self.with_long_library_names = False - self.copy_shared_index_config = False - self.idea_jdk_version = None - self.regenerate_with_project_update = False - self.project_update_targets = [] - self.project_update_kind = None - - @staticmethod - def consumer(): - return [ - core.yarg.ArgConsumer( - ['-r', '--project-root'], - help='IntelliJ IDEA project root path', - hook=core.yarg.SetValueHook('idea_project_root'), - group=IdeaOptions.IDEA_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['-P', '--project-output'], - help='IntelliJ IDEA project root path. Please, use instead of -r', - hook=core.yarg.SetValueHook('idea_project_root'), - group=IdeaOptions.IDEA_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['-l', '--local'], - help='Only recurse reachable projects are idea modules', - hook=core.yarg.SetConstValueHook('local', True), - group=IdeaOptions.IDEA_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--group-modules'], - help='Group idea modules according to paths: (tree, flat)', - hook=core.yarg.SetValueHook('group_modules', values=('tree', 'flat')), - group=IdeaOptions.IDEA_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['-n', '--dry-run'], - help='Emulate create project, but do nothing', - hook=core.yarg.SetConstValueHook('dry_run', True), - group=IdeaOptions.IDEA_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--ymake-bin'], help='Path to ymake binary', hook=core.yarg.SetValueHook('ymake_bin'), visible=False - ), - core.yarg.ArgConsumer( - ['--iml-in-project-root'], - help='Store ".iml" files in project root tree(stores in source root tree by default)', - hook=core.yarg.SetConstValueHook('iml_in_project_root', True), - ), - core.yarg.ArgConsumer( - ['--iml-keep-relative-paths'], - help='Keep relative paths in ".iml" files (works with --iml-in-project-root)', - hook=core.yarg.SetConstValueHook('iml_keep_relative_paths', True), - ), - core.yarg.ArgConsumer( - ['--idea-files-root'], - help='Root for .ipr and .iws files', - hook=core.yarg.SetValueHook('idea_files_root'), - ), - core.yarg.ArgConsumer( - ['--project-name'], - help='Idea project name (.ipr and .iws file)', - hook=core.yarg.SetValueHook('project_name'), - ), - core.yarg.ArgConsumer( - ['--ascetic'], - help='Create the minimum set of project settings', - hook=core.yarg.SetConstValueHook('minimal', True), - ), - core.yarg.ArgConsumer( - ['--directory-based'], - help='Create project in actual (directory based) format', - hook=core.yarg.SetConstValueHook('directory_based', True), - visible=False, - ), - core.yarg.ArgConsumer( - ['--omit-test-data'], - help='Do not export test_data', - hook=core.yarg.SetConstValueHook('omit_test_data', True), - ), - core.yarg.ArgConsumer( - ['--with-content-root-modules'], - help='Generate content root modules', - hook=core.yarg.SetConstValueHook('with_content_root_modules', True), - ), - core.yarg.ArgConsumer( - ['--external-content-root-module'], - help='Add external content root modules', - hook=core.yarg.SetAppendHook('external_content_root_modules'), - ), - core.yarg.ArgConsumer( - ['--generate-junit-run-configurations'], - help='Generate run configuration for junit tests', - hook=core.yarg.SetConstValueHook('generate_tests_run', True), - ), - core.yarg.ArgConsumer( - ['--generate-tests-for-dependencies'], - help='Generate tests for PEERDIR dependencies', - hook=core.yarg.SetConstValueHook('generate_tests_for_deps', True), - ), - core.yarg.ArgConsumer( - ['--separate-tests-modules'], - help='Do not merge tests modules with their own libraries', - hook=core.yarg.SetConstValueHook('separate_tests_modules', True), - ), - core.yarg.ArgConsumer( - ['--auto-exclude-symlinks'], - help='Add all symlink-dirs in modules to exclude dirs', - hook=core.yarg.SetConstValueHook('auto_exclude_symlinks', True), - ), - core.yarg.ArgConsumer( - ['--exclude-dirs'], - help='Exclude dirs with specific names from all modules', - hook=core.yarg.SetAppendHook('exclude_dirs'), - ), - core.yarg.ArgConsumer( - ['--with-common-jvm-args-in-junit-template'], - help='Add common JVM_ARGS flags to default junit template', - hook=core.yarg.SetConstValueHook('with_common_jvm_args_in_junit_template', True), - ), - core.yarg.ArgConsumer( - ['--with-long-library-names'], - help='Generate long library names', - hook=core.yarg.SetConstValueHook('with_long_library_names', True), - ), - core.yarg.ArgConsumer( - ['--copy-shared-index-config'], - help='Copy project config for Shared Indexes if exist', - hook=core.yarg.SetConstValueHook('copy_shared_index_config', True), - ), - core.yarg.ArgConsumer( - ['--idea-jdk-version'], - help='Project JDK version', - hook=core.yarg.SetValueHook('idea_jdk_version'), - ), - core.yarg.ArgConsumer( - ['-U', '--regenerate-with-project-update'], - help='Run `ya project update` upon regeneration from Idea', - group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP, - hook=core.yarg.SetConstValueHook('regenerate_with_project_update', True), - ), - core.yarg.ArgConsumer( - ['--project-update-targets'], - help='Run `ya project update` for this dirs upon regeneration from Idea', - hook=core.yarg.SetAppendHook('project_update_targets'), - group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP, - visible=HelpLevel.ADVANCED, - ), - core.yarg.ArgConsumer( - ['--project-update-kind'], - help='Type of a project to use in `ya project update` upon regernation from Idea', - hook=core.yarg.SetValueHook('project_update_kind'), - group=IdeaOptions.IDE_PLUGIN_INTEGRATION_GROUP, - visible=HelpLevel.ADVANCED, - ), - core.yarg.ConfigConsumer('idea_project_root'), - core.yarg.ConfigConsumer('local'), - core.yarg.ConfigConsumer('group_modules'), - core.yarg.ConfigConsumer('dry_run'), - core.yarg.ConfigConsumer('iml_in_project_root'), - core.yarg.ConfigConsumer('iml_keep_relative_paths'), - core.yarg.ConfigConsumer('idea_files_root'), - core.yarg.ConfigConsumer('project_name'), - core.yarg.ConfigConsumer('minimal'), - core.yarg.ConfigConsumer('directory_based'), - core.yarg.ConfigConsumer('omit_test_data'), - core.yarg.ConfigConsumer('with_content_root_modules'), - core.yarg.ConfigConsumer('external_content_root_modules'), - core.yarg.ConfigConsumer('generate_tests_run'), - core.yarg.ConfigConsumer('generate_tests_for_deps'), - core.yarg.ConfigConsumer('separate_tests_modules'), - core.yarg.ConfigConsumer('auto_exclude_symlinks'), - core.yarg.ConfigConsumer('exclude_dirs'), - core.yarg.ConfigConsumer('with_common_jvm_args_in_junit_template'), - core.yarg.ConfigConsumer('with_long_library_names'), - core.yarg.ConfigConsumer('copy_shared_index_config'), - core.yarg.ConfigConsumer('idea_jdk_version'), - core.yarg.ConfigConsumer('regenarate_with_project_update'), - core.yarg.ConfigConsumer('project_update_targets'), - core.yarg.ConfigConsumer('project_update_kind'), - ] - - def postprocess(self): - if self.idea_project_root is None: - raise core.yarg.ArgsValidatingException('Idea project root(-r, --project-root) must be specified.') - - self.idea_project_root = os.path.abspath(self.idea_project_root) - - if self.iml_keep_relative_paths and not self.iml_in_project_root: - raise core.yarg.ArgsValidatingException( - '--iml-keep-relative-paths can be used only with --iml-in-project-root' - ) - - if self.generate_tests_run and not self.directory_based: - raise core.yarg.ArgsValidatingException( - 'run configurations may be generated only for directory-based project' - ) - - for p in self.exclude_dirs: - if os.path.isabs(p): - raise core.yarg.ArgsValidatingException('Absolute paths are not allowed in --exclude-dirs') - - -class GradleOptions(core.yarg.Options): - GRADLE_OPT_GROUP = core.yarg.Group('Gradle project options', 0) - - def __init__(self): - self.gradle_name = None - - @staticmethod - def consumer(): - return [ - core.yarg.ArgConsumer( - ['--gradle-name'], - help='Set project name manually', - hook=core.yarg.SetValueHook('gradle_name'), - group=GradleOptions.GRADLE_OPT_GROUP, - ) - ] - - -class PycharmOptions(core.yarg.Options): - PYCHARM_OPT_GROUP = core.yarg.Group('Pycharm project options', 0) - PYTHON_WRAPPER_NAME = 'pycharm_python_wrapper' - - def __init__(self): - self.only_generate_wrapper = False - self.wrapper_name = PycharmOptions.PYTHON_WRAPPER_NAME - self.list_ide = False - self.ide_version = None - - @staticmethod - def consumer(): - return [ - core.yarg.ArgConsumer( - ['--only-generate-wrapper'], - help="Don not generate Pycharm project, only wrappers", - hook=core.yarg.SetConstValueHook('only_generate_wrapper', True), - group=PycharmOptions.PYCHARM_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--wrapper-name'], - help='Name of generated python wrapper. Use `python` for manual adding wrapper as Python SDK.', - hook=core.yarg.SetValueHook('wrapper_name'), - group=PycharmOptions.PYCHARM_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--list-ide'], - help='List available JB IDE for patching SDK list.', - hook=core.yarg.SetConstValueHook('list_ide', True), - group=PycharmOptions.PYCHARM_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--ide-version'], - help='Change IDE version for patching SDK list. Available IDE: {}'.format( - ", ".join(ide.pycharm.find_available_ide()) - ), - hook=core.yarg.SetValueHook('ide_version'), - group=PycharmOptions.PYCHARM_OPT_GROUP, - ), - ] - - def postprocess(self): - if PycharmOptions.PYTHON_WRAPPER_NAME != self.wrapper_name and not self.only_generate_wrapper: - raise core.yarg.ArgsValidatingException( - "Custom wrapper name can be used with option --only-generate-wrapper" - ) - - -MSVS_OPTS = ide.msvs.MSVS_OPTS + [ide.ide_common.YaExtraArgsOptions(), core.common_opts.YaBin3Options()] - - -def gen_msvs_solution(params): - impl = ide.msvs_lite if params.lite else ide.msvs - return impl.gen_msvs_solution(params) - - -def get_description(text, ref_name): - if app_config.in_house: - ref = { - "c": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#c", - "golang": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#golang", - "multi": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#multi", - "python": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#python", - "typescript": "https://docs.yandex-team.ru/ya-make/usage/ya_ide/vscode#typescript", - }[ref_name] - return "{}\nDocs: [[c:dark-cyan]]{}[[rst]]".format(text, ref) - else: - return text - - -class IdeYaHandler(core.yarg.CompositeHandler): - description = 'Generate project for IDE' - - def __init__(self): - core.yarg.CompositeHandler.__init__(self, description=self.description) - self['msvs'] = core.yarg.OptsHandler( - action=app.execute(gen_msvs_solution), - description='[[imp]]ya ide msvs[[rst]] is deprecated, please use clangd-based tooling instead', - opts=MSVS_OPTS, - examples=[ - core.yarg.UsageExample( - '{prefix} util/generic util/datetime', - 'Generate solution for util/generic, util/datetime and all their dependencies', - ), - core.yarg.UsageExample('{prefix} -P Output', 'Generate solution in Output directory'), - core.yarg.UsageExample('{prefix} -T my_solution', 'Generate solution titled my_solution.sln'), - ], - visible=(pm.my_platform() == 'win32'), - ) - self['clion'] = core.yarg.OptsHandler( - action=app.execute(ide.clion2016.do_clion), - description='[[imp]]ya ide clion[[rst]] is deprecated, please use clangd-based tooling instead', - opts=ide.ide_common.ide_via_ya_make_opts() - + [ - CLionOptions(), - TidyOptions(), - core.common_opts.YaBin3Options(), - ], - ) - - self['idea'] = core.yarg.OptsHandler( - action=app.execute(ide.idea.do_idea), - description='Generate stub for IntelliJ IDEA', - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - ide.ide_common.IdeYaMakeOptions(), - ide.ide_common.YaExtraArgsOptions(), - IdeaOptions(), - core.common_opts.OutputStyleOptions(), - core.common_opts.CrossCompilationOptions(), - core.common_opts.PrintStatisticsOptions(), - build.build_opts.ContinueOnFailOptions(), - build.build_opts.YMakeDebugOptions(), - build.build_opts.BuildThreadsOptions(build_threads=None), - build.build_opts.DistCacheOptions(), - build.build_opts.FlagsOptions(), - build.build_opts.IgnoreRecursesOptions(), - build.build_opts.ContentUidsOptions(), - build.build_opts.ExecutorOptions(), - build.build_opts.CustomFetcherOptions(), - build.build_opts.SandboxAuthOptions(), - core.common_opts.YaBin3Options(), - ], - unknown_args_as_free=True, - ) - ide_gradle_opts = ide.ide_common.ide_minimal_opts(targets_free=True) + [ - ide.ide_common.YaExtraArgsOptions(), - GradleOptions(), - core.yarg.ShowHelpOptions(), - build.build_opts.FlagsOptions(), - build.build_opts.CustomFetcherOptions(), - build.build_opts.SandboxAuthOptions(), - core.common_opts.CrossCompilationOptions(), - build.build_opts.ToolsOptions(), - build.build_opts.BuildTypeOptions('release'), - build.build_opts.JavaSpecificOptions(), - ] - if six.PY2: - self['gradle'] = core.yarg.OptsHandler( - action=app.execute( - lambda *a, **k: None, - handler_python_major_version=3, - ), - description='Generate gradle for project', - opts=ide_gradle_opts, - visible=False, - ) - if six.PY3: - self['gradle'] = core.yarg.OptsHandler( - action=app.execute( - ide.gradle.do_gradle, - handler_python_major_version=3, - ), - description='Generate gradle for project', - opts=ide_gradle_opts, - visible=False, - ) - self['qt'] = core.yarg.OptsHandler( - action=app.execute(self._choose_qt_handler), - description='[[imp]]ya ide qt[[rst]] is deprecated, please use clangd-based tooling instead', - opts=ide.qt.QT_OPTS + [core.common_opts.YaBin3Options()], - ) - self['goland'] = core.yarg.OptsHandler( - action=app.execute(ide.goland.do_goland), - description='Generate stub for Goland', - opts=ide.ide_common.ide_via_ya_make_opts() - + [ - ide.goland.GolandOptions(), - core.common_opts.YaBin3Options(), - ], - ) - self['pycharm'] = core.yarg.OptsHandler( - action=app.execute(ide.pycharm.do_pycharm), - description='Generate PyCharm project.', - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - PycharmOptions(), - ide.ide_common.IdeYaMakeOptions(), - ide.ide_common.YaExtraArgsOptions(), - build.build_opts.DistCacheOptions(), - core.common_opts.YaBin3Options(), - ], - visible=(pm.my_platform() != 'win32'), - ) - self['vscode-clangd'] = core.yarg.OptsHandler( - action=app.execute(ide.vscode_clangd.gen_vscode_workspace), - description=get_description('Generate VSCode clangd C++ project.', ref_name='c'), - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - ide.vscode_clangd.VSCodeClangdOptions(), - ide.ide_common.YaExtraArgsOptions(), - bcd.CompilationDatabaseOptions(), - build.build_opts.FlagsOptions(), - build.build_opts.OutputOptions(), - build.build_opts.BuildThreadsOptions(build_threads=None), - build.build_opts.ArcPrefetchOptions(), - build.build_opts.ContentUidsOptions(), - build.build_opts.ToolsOptions(), - core.common_opts.YaBin3Options(), - ], - visible=(pm.my_platform() != 'win32'), - ) - self['vscode-go'] = core.yarg.OptsHandler( - action=app.execute(ide.vscode_go.gen_vscode_workspace), - description=get_description('Generate VSCode Go project.', ref_name='golang'), - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - ide.vscode_go.VSCodeGoOptions(), - ide.ide_common.YaExtraArgsOptions(), - build.build_opts.FlagsOptions(), - build.build_opts.BuildThreadsOptions(build_threads=None), - build.build_opts.ArcPrefetchOptions(), - build.build_opts.ContentUidsOptions(), - build.build_opts.ToolsOptions(), - core.common_opts.YaBin3Options(), - ], - ) - self['vscode-py'] = core.yarg.OptsHandler( - action=app.execute(ide.vscode_py.gen_vscode_workspace), - description=get_description('Generate VSCode Python project.', ref_name='python'), - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - ide.vscode_py.VSCodePyOptions(), - ide.ide_common.YaExtraArgsOptions(), - build.build_opts.FlagsOptions(), - build.build_opts.BuildThreadsOptions(build_threads=None), - build.build_opts.ArcPrefetchOptions(), - build.build_opts.ContentUidsOptions(), - build.build_opts.ToolsOptions(), - core.common_opts.YaBin3Options(), - ], - visible=(pm.my_platform() != 'win32'), - ) - self['vscode-ts'] = core.yarg.OptsHandler( - action=app.execute(ide.vscode_py.gen_vscode_workspace), - description=get_description('Generate VSCode TypeScript project.', ref_name='typescript'), - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - ide.vscode_py.VSCodePyOptions(), - ide.ide_common.YaExtraArgsOptions(), - build.build_opts.FlagsOptions(), - build.build_opts.BuildThreadsOptions(build_threads=None), - build.build_opts.ArcPrefetchOptions(), - build.build_opts.ContentUidsOptions(), - build.build_opts.ToolsOptions(), - core.common_opts.YaBin3Options(), - ], - visible=(pm.my_platform() != 'win32'), - ) - self['vscode'] = core.yarg.OptsHandler( - action=app.execute(ide.vscode_all.gen_vscode_workspace), - description=get_description('Generate VSCode multi-language project.', ref_name='multi'), - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - ide.vscode.opts.VSCodeAllOptions(), - ide.ide_common.YaExtraArgsOptions(), - bcd.CompilationDatabaseOptions(), - build.build_opts.ArcPrefetchOptions(prefetch=True, visible=False), - build.build_opts.FlagsOptions(), - build.build_opts.OutputOptions(), - build.build_opts.BuildThreadsOptions(build_threads=None), - build.build_opts.ContentUidsOptions(), - build.build_opts.ToolsOptions(), - core.common_opts.YaBin3Options(), - ], - ) - self['vscode-ts'] = core.yarg.OptsHandler( - action=app.execute(ide.vscode_ts.gen_vscode_workspace), - description=get_description('Generate VSCode TypeScript project.', ref_name='typescript'), - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - ide.vscode_ts.VSCodeTypeScriptOptions(), - core.common_opts.YaBin3Options(), - ], - visible=False, # FIXME: remove when ready for public release - ) - self['venv'] = core.yarg.OptsHandler( - action=app.execute(ide.venv.do_venv), - description='Create or update python venv', - opts=ide.ide_common.ide_minimal_opts(targets_free=True) - + [ - build.build_opts.BuildTypeOptions('release'), - build.build_opts.BuildThreadsOptions(build_threads=None), - build.build_opts.ExecutorOptions(), - build.build_opts.FlagsOptions(), - build.build_opts.IgnoreRecursesOptions(), - build.build_opts.RebuildOptions(), - core.common_opts.BeVerboseOptions(), - core.common_opts.CrossCompilationOptions(), - ide.ide_common.YaExtraArgsOptions(), - ide.venv.VenvOptions(), - core.common_opts.YaBin3Options(), - ], - visible=(pm.my_platform() != 'win32'), - ) - if app_config.in_house: - self['fix-jb-fsnotifier'] = core.yarg.OptsHandler( - action=app.execute(devtools.ya.ide.fsnotifier.fix_fsnotifier), - description='Replace fsnotifier for JB IDEs.', - opts=[ - devtools.ya.ide.fsnotifier.FixFsNotifierOptions(), - core.common_opts.ShowHelpOptions(), - core.common_opts.DumpDebugOptions(), - core.common_opts.AuthOptions(), - core.common_opts.YaBin3Options(), - ], - ) - - @staticmethod - def _choose_qt_handler(params): - if params.run: - ide.qt.run_qtcreator(params) - elif params.remote_host: - ide.remote_ide_qt.generate_remote_project(params) - else: - ide.qt.gen_qt_project(params) diff --git a/devtools/ya/handlers/ide/ya.make b/devtools/ya/handlers/ide/ya.make deleted file mode 100644 index c0802be5352..00000000000 --- a/devtools/ya/handlers/ide/ya.make +++ /dev/null @@ -1,31 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.ide - __init__.py -) - -PEERDIR( - devtools/ya/app - devtools/ya/build/build_opts - devtools/ya/core - devtools/ya/core/config - devtools/ya/core/yarg - devtools/ya/ide - devtools/ya/yalibrary/platform_matcher -) - -IF (NOT YA_OPENSOURCE) - PEERDIR( - devtools/ya/ide/fsnotifier - ) -ENDIF() - -END() - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/java/__init__.py b/devtools/ya/handlers/java/__init__.py deleted file mode 100644 index 5c4c7c44e3c..00000000000 --- a/devtools/ya/handlers/java/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import absolute_import -from . import helpers - -import app -import core.yarg -from build import build_opts -import devtools.ya.test.opts as test_opts - - -def default_options(): - return [ - build_opts.BuildTargetsOptions(with_free=True), - build_opts.BeVerboseOptions(), - build_opts.ShowHelpOptions(), - build_opts.YMakeDebugOptions(), - build_opts.YMakeBinOptions(), - build_opts.YMakeRetryOptions(), - build_opts.FlagsOptions(), - ] - - -class JavaYaHandler(core.yarg.CompositeHandler): - def __init__(self): - super(JavaYaHandler, self).__init__(description='Java build helpers') - - self['dependency-tree'] = core.yarg.OptsHandler( - action=app.execute(action=helpers.print_ymake_dep_tree), - description='Print dependency tree', - opts=default_options() + [build_opts.BuildTypeOptions('release')], - visible=True, - ) - self['classpath'] = core.yarg.OptsHandler( - action=app.execute(action=helpers.print_classpath), - description='Print classpath', - opts=default_options() + [build_opts.BuildTypeOptions('release')], - visible=True, - ) - self['test-classpath'] = core.yarg.OptsHandler( - action=app.execute(action=helpers.print_test_classpath), - description='Print run classpath for test module', - opts=default_options() + [test_opts.RunTestOptions()], - visible=True, - ) - self['find-all-paths'] = core.yarg.OptsHandler( - action=app.execute(action=helpers.find_all_paths), - description='Find all PEERDIR paths of between two targets', - opts=default_options() + [build_opts.FindPathOptions()], - visible=True, - ) diff --git a/devtools/ya/handlers/java/helpers.py b/devtools/ya/handlers/java/helpers.py deleted file mode 100644 index 8a5a9cd35a5..00000000000 --- a/devtools/ya/handlers/java/helpers.py +++ /dev/null @@ -1,250 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -import time -import sys - -import core.yarg -import build.build_opts as build_opts -import build.graph as build_graph -import build.ya_make as ya_make -from build.build_facade import gen_managed_dep_tree, gen_targets_classpath -from exts.tmp import temp_dir -import yalibrary.formatter as yaformatter - -from jbuild.gen import base -from jbuild.gen import consts -from six.moves import map - - -def fix_windows(path): - return path.replace('\\', '/') - - -def get_java_ctx_with_tests(opts): - jopts = core.yarg.merge_opts(build_opts.ya_make_options(free_build_targets=True)).params() - jopts.dump_sources = True - jopts.create_symlinks = False - jopts.__dict__.update(opts.__dict__) - jopts.debug_options.append('x') - jopts.flags["YA_IDE_IDEA"] = "yes" - - import app_ctx # XXX: via args - - _, tests, _, ctx, _ = build_graph.build_graph_and_tests( - jopts, check=True, ev_listener=ya_make.get_print_listener(jopts, app_ctx.display), display=app_ctx.display - ) - - return ctx, tests - - -def get_java_ctx(opts): - ctx, _ = get_java_ctx_with_tests(opts) - return ctx - - -NORMAL = '[[imp]]{path}[[rst]]' -EXCLUDED = '[[unimp]]{path} (omitted because of [[c:red]]EXCLUDE[[unimp]])[[rst]]' -CONFLICT = '[[unimp]]{path} (omitted because of [[c:yellow]]conflict with {conflict_resolution}[[unimp]])[[rst]]' -MANAGED = '[[imp]]{path}[[unimp]] (replaced from [[c:blue]]{orig}[[unimp]] because of [[c:blue]]DEPENDENCY_MANAGEMENT[[unimp]])[[rst]]' -IGNORED = '[[unimp]]{path} (omitted as contrib proxy library)[[rst]]' -DUPLICATE = '[[unimp]]{path} (*)[[rst]]' -DIRECT_MANAGED = '[[imp]]{path}[[unimp]] (replaced from [[c:blue]]unversioned[[unimp]] because of [[c:blue]]DEPENDENCY_MANAGEMENT[[unimp]])[[rst]]' -DIRECT_DEFAULT = ( - '[[imp]]{path}[[unimp]] (replaced from [[c:magenta]]unversioned[[unimp]] to [[c:magenta]]default[[unimp]])[[rst]]' -) - - -def arrow_str(length): - s = '' - - if length: - s += '| ' * (length - 1) + '|-->' - - return '[[unimp]]' + s + '[[rst]]' - - -def node_str(dep, excluded=False, expanded_above=False): - if expanded_above: - return DUPLICATE.format(path=dep.path) - - elif excluded: - return EXCLUDED.format(path=dep.path) - - elif dep.omitted_for_conflict: - return CONFLICT.format(path=dep.path, conflict_resolution=base.basename_unix_path(dep.conflict_resolution_path)) - - elif dep.replaced_for_dependency_management: - return MANAGED.format(path=dep.path, orig=base.basename_unix_path(dep.old_path_for_dependency_management)) - - elif dep.is_managed: - return DIRECT_MANAGED.format(path=dep.path) - - elif dep.is_default: - return DIRECT_DEFAULT.format(path=dep.path) - - else: - return NORMAL.format(path=dep.path) - - -def graph_line(depth, dep, excluded=False, expanded_above=False): - return arrow_str(depth) + node_str(dep, excluded=excluded, expanded_above=expanded_above) - - -def raise_not_a_java_path(path): - raise WrongInputException('{} is not a java module'.format(path)) - - -class WrongInputException(Exception): - mute = True - - -def print_ymake_dep_tree(opts): - with temp_dir() as tmp: - res, evlog = gen_managed_dep_tree( - build_root=tmp, - build_type=opts.build_type, - build_targets=opts.abs_targets, - debug_options=opts.debug_options, - flags=opts.flags, - ymake_bin=opts.ymake_bin, - ) - - import app_ctx - - ev_listener = ya_make.get_print_listener(opts, app_ctx.display) - for ev in evlog: - ev_listener(ev) - - formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty()) - print(formatter.format_message(res.stdout)) - return any('Type' in ev and ev['Type'] == 'Error' for ev in evlog) - - -def print_classpath(opts): - opts.flags['TRAVERSE_RECURSE'] = 'no' - opts.flags['TRAVERSE_RECURS_FOR_TEST'] = 'no' - with temp_dir() as tmp: - res, evlog = gen_targets_classpath( - build_root=tmp, - build_type=opts.build_type, - build_targets=opts.abs_targets, - debug_options=opts.debug_options, - flags=opts.flags, - ymake_bin=opts.ymake_bin, - ) - - import app_ctx - - ev_listener = ya_make.get_print_listener(opts, app_ctx.display) - for ev in evlog: - ev_listener(ev) - - formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty()) - print(formatter.format_message(res.stdout)) - return any('Type' in ev and ev['Type'] == 'Error' for ev in evlog) - - -def print_test_classpath(opts): - opts.flags['IGNORE_JAVA_DEPENDENCIES_CONFIGURATION'] = 'yes' - opts.run_tests = 3 - ctx, tests = get_java_ctx_with_tests(opts) - remaining_roots = set(map(fix_windows, opts.rel_targets)) - - formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty()) - for test in tests: - if test.project_path not in remaining_roots: - continue - classpath = test.get_classpath() - if classpath is not None: - classpath = [item[len(consts.BUILD_ROOT) + 1 :] for item in classpath] - print(formatter.format_message("[[imp]]{}[[rst]]:\n\t{}".format(test.project_path, '\n\t'.join(classpath)))) - remaining_roots.discard(test.project_path) - - for path in remaining_roots: - print(formatter.format_message("[[imp]]{}[[rst]] is not a Java test".format(path))) - - -def iter_all_routes(ctx, src_path, dest_path, route_callback, max_dist=None): - achievable = {} - - def dest_achievable_from(path): - if path not in achievable: - achievable[path] = path == dest_path or any(dest_achievable_from(d.path) for d in ctx.by_path[path].deps) - - return achievable[path] - - stack = [] - - def visit(path): - stack.append(path) - - if path == dest_path: - route_callback(stack) - stack.pop() - return # no loops - - if max_dist is not None and len(stack) - 1 >= max_dist: - stack.pop() - return - - for dep in ctx.by_path[path].deps: - dep_path = dep.path - - if dest_achievable_from(dep_path): - visit(dep_path) - - stack.pop() - - if dest_achievable_from(src_path): - visit(src_path) - - -def print_route(route): - dist = len(route) - 1 - - if dist < 4: - s = '[[c:green]]{}[[rst]]: '.format(dist) - elif dist < 7: - s = '[[c:yellow]]{}[[rst]]: '.format(dist) - else: - s = '[[c:red]]{}[[rst]]: '.format(dist) - - s += '\n[[unimp]]-->'.join(['[[imp]]' + p for p in route]) - - return s - - -def find_all_paths(opts): - start_time = time.time() - opts.flags['IGNORE_JAVA_DEPENDENCIES_CONFIGURATION'] = 'yes' - ctx = get_java_ctx(opts) - - if len(opts.rel_targets) != 2: - raise WrongInputException( - 'Expected: <from> <to>\n' + 'Got: {}'.format(' '.join(['<' + p + '>' for p in opts.rel_targets])) - ) - - src_path = fix_windows(opts.rel_targets[0]) - dest_path = fix_windows(opts.rel_targets[1]) - - if src_path not in ctx.by_path: - raise_not_a_java_path(src_path) - - if dest_path not in ctx.by_path: - raise_not_a_java_path(dest_path) - - count = [0] - - formatter = yaformatter.new_formatter(is_tty=sys.stdout.isatty()) - - def route_callback(route): - print(formatter.format_message(print_route(route))) - count[0] += 1 - - try: - iter_all_routes(ctx, src_path, dest_path, route_callback, max_dist=opts.max_dist) - finally: - report = 'Found [[c:{}]]{}[[rst]] paths in [[c:yellow]]{}[[rst]] seconds'.format( - 'green' if count[0] else 'red', count[0], time.time() - start_time - ) - print(formatter.format_message(report)) diff --git a/devtools/ya/handlers/java/ya.make b/devtools/ya/handlers/java/ya.make deleted file mode 100644 index 96140c1bf13..00000000000 --- a/devtools/ya/handlers/java/ya.make +++ /dev/null @@ -1,24 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.java - __init__.py - helpers.py -) - -PEERDIR( - devtools/ya/app - devtools/ya/core/yarg - devtools/ya/build - devtools/ya/build/build_opts - devtools/ya/test/opts -) - -END() - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/make/__init__.py b/devtools/ya/handlers/make/__init__.py deleted file mode 100644 index e9d774f5629..00000000000 --- a/devtools/ya/handlers/make/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import absolute_import -from build.build_handler import do_ya_make -from build.build_opts import ya_make_options - -import core.yarg - -import app - - -class MakeYaHandler(core.yarg.OptsHandler): - description = 'Build and run tests\nTo see more help use [[imp]]-hh[[rst]]/[[imp]]-hhh[[rst]]' - - def __init__(self): - core.yarg.OptsHandler.__init__( - self, - action=app.execute(action=do_ya_make), - examples=[ - core.yarg.UsageExample('{prefix} -r', 'Build current directory in release mode', good_looking=100), - core.yarg.UsageExample( - '{prefix} -t -j16 library', 'Build and test library with 16 threads', good_looking=99 - ), - core.yarg.UsageExample( - '{prefix} --checkout -j0', 'Checkout absent directories without build', good_looking=98 - ), - ], - description=self.description, - opts=ya_make_options( - free_build_targets=True, - strip_idle_build_results=True, - ), - visible=True, - ) diff --git a/devtools/ya/handlers/make/ya.make b/devtools/ya/handlers/make/ya.make deleted file mode 100644 index 275750e1f75..00000000000 --- a/devtools/ya/handlers/make/ya.make +++ /dev/null @@ -1,26 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.make - __init__.py -) - -PEERDIR( - devtools/ya/app - devtools/ya/build - devtools/ya/build/build_opts - devtools/ya/core/yarg -) - -END() - -RECURSE( - tests -) - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/maven_import/__init__.py b/devtools/ya/handlers/maven_import/__init__.py deleted file mode 100644 index f1296c0c83e..00000000000 --- a/devtools/ya/handlers/maven_import/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import absolute_import -import core.yarg -import jbuild.maven.maven_import as mi -import core.common_opts as common_opts -import build.build_opts as build_opts -import app -import app_config - -from core.yarg.help_level import HelpLevel - - -class MavenImportYaHandler(core.yarg.OptsHandler): - description = 'Import specified artifact from remote maven repository to arcadia/contrib/java.' - visible = app_config.in_house - - def __init__(self): - super(MavenImportYaHandler, self).__init__( - action=app.execute(mi.do_import), - opts=[ - build_opts.MavenImportOptions(visible=HelpLevel.BASIC), - build_opts.BuildThreadsOptions(build_threads=None), - common_opts.OutputStyleOptions(), - common_opts.TransportOptions(), - common_opts.ShowHelpOptions(), - build_opts.YMakeBinOptions(), - build_opts.CustomFetcherOptions(), - build_opts.ToolsOptions(), - ] - + build_opts.checkout_options(), - description=self.description, - ) diff --git a/devtools/ya/handlers/maven_import/ya.make b/devtools/ya/handlers/maven_import/ya.make deleted file mode 100644 index c2e16c6d357..00000000000 --- a/devtools/ya/handlers/maven_import/ya.make +++ /dev/null @@ -1,28 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.maven_import - __init__.py -) - -PEERDIR( - devtools/ya/app - devtools/ya/build/build_opts - devtools/ya/core - devtools/ya/core/yarg - devtools/ya/jbuild - devtools/ya/yalibrary/tools -) - -END() - -RECURSE( - tests -) - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/package/__init__.py b/devtools/ya/handlers/package/__init__.py deleted file mode 100644 index adaf2980f13..00000000000 --- a/devtools/ya/handlers/package/__init__.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import absolute_import -import logging - -import app -import core.yarg -import core.common_opts -import build.ya_make -import build.targets_deref -import build.build_opts -import devtools.ya.test.opts as test_opts - -import package.docker -import package.packager -import devtools.ya.handlers.package.opts as package_opts -from core.yarg.help_level import HelpLevel - -logger = logging.getLogger(__name__) - - -class PackageYaHandler(core.yarg.OptsHandler): - description = """Build package using json package description in the release build type by default. -For more info see https://docs.yandex-team.ru/ya-make/usage/ya_package""" - - def __init__(self): - super(PackageYaHandler, self).__init__( - action=app.execute(package.packager.do_package, respawn=app.RespawnType.OPTIONAL), - description=self.description, - examples=[ - core.yarg.UsageExample( - cmd='{prefix} <path to json description>', - description='Create tarball package from json description', - ) - ], - opts=[ - package_opts.PackageOperationalOptions(), - package_opts.PackageCustomizableOptions(), - package_opts.InterimOptions(), - core.common_opts.LogFileOptions(), - core.common_opts.EventLogFileOptions(), - build.build_opts.BuildTypeOptions('release'), - build.build_opts.BuildThreadsOptions(build_threads=None), - core.common_opts.CrossCompilationOptions(), - build.build_opts.ArcPrefetchOptions(), - build.build_opts.ContentUidsOptions(), - build.build_opts.KeepTempsOptions(), - build.build_opts.RebuildOptions(), - build.build_opts.StrictInputsOptions(), - build.build_opts.DumpReportOptions(), - build.build_opts.OutputOptions(), - build.build_opts.AuthOptions(), - build.build_opts.YMakeDumpGraphOptions(), - build.build_opts.YMakeDebugOptions(), - build.build_opts.YMakeBinOptions(), - build.build_opts.YMakeRetryOptions(), - build.build_opts.ExecutorOptions(), - build.build_opts.ForceDependsOptions(), - build.build_opts.IgnoreRecursesOptions(), - core.common_opts.CustomSourceRootOptions(), - core.common_opts.CustomBuildRootOptions(), - core.common_opts.ShowHelpOptions(), - core.common_opts.BeVerboseOptions(), - core.common_opts.HtmlDisplayOptions(), - core.common_opts.CommonUploadOptions(), - build.build_opts.SandboxUploadOptions(ssh_key_option_name="--ssh-key", visible=HelpLevel.BASIC), - build.build_opts.MDSUploadOptions(visible=HelpLevel.BASIC), - core.common_opts.TransportOptions(), - build.build_opts.CustomFetcherOptions(), - build.build_opts.DistCacheOptions(), - build.build_opts.FlagsOptions(), - build.build_opts.PGOOptions(), - test_opts.RunTestOptions(), - test_opts.DebuggingOptions(), - # strip_idle_build_results must be False to avoid removal of build nodes which are - # reachable due RECURSE and used in package, but not required for tests - test_opts.DepsOptions(strip_idle_build_results=False), - test_opts.FileReportsOptions(), - test_opts.FilteringOptions(test_size_filters=None), - test_opts.PytestOptions(), - test_opts.JUnitOptions(), - test_opts.RuntimeEnvironOptions(), - test_opts.TestToolOptions(), - test_opts.UidCalculationOptions(cache_tests=False), - core.common_opts.YaBin3Options(), - ] - + build.build_opts.distbs_options() - + build.build_opts.checkout_options() - + build.build_opts.svn_checkout_options(), - ) diff --git a/devtools/ya/handlers/package/opts/__init__.py b/devtools/ya/handlers/package/opts/__init__.py deleted file mode 100644 index e65532ef6f5..00000000000 --- a/devtools/ya/handlers/package/opts/__init__.py +++ /dev/null @@ -1,691 +0,0 @@ -import collections -import logging - -import build.build_opts -import core.yarg -import devtools.ya.test.opts as test_opts -from devtools.ya.package import const - -logger = logging.getLogger(__name__) - - -COMMON_SUBGROUP = core.yarg.Group('Common', 1) -TAR_SUBGROUP = core.yarg.Group('Tar', 2) -DEB_SUBGROUP = core.yarg.Group('Debian', 3) -DOCKER_SUBGROUP = core.yarg.Group('Docker', 4) -AAR_SUBGROUP = core.yarg.Group('Aar', 5) -RPM_SUBGROUP = core.yarg.Group('Rpm', 6) -NPM_SUBGROUP = core.yarg.Group('Npm', 7) -PYTHON_WHEEL_SUBGROUP = core.yarg.Group('Python wheel', 8) - - -class PackageOperationalOptions(core.yarg.Options): - def __init__(self): - self.artifactory_password_path = None - self.build_debian_scripts = False - self.build_only = False - self.change_log = None - self.cleanup = True - self.codec = None - self.convert = None - self.custom_data_root = None - self.custom_tests_data_root = None - self.debian_distribution = 'unstable' - self.debian_upload_token = None # please, do not remove, we really need it in opensource nebius ya - self.docker_no_cache = False - self.docker_push_image = False - self.docker_remote_image_version = None - self.docker_use_remote_cache = False - self.dump_build_targets = None - self.dump_inputs = None - self.ignore_fail_tests = False - self.list_codecs = False - self.nanny_release = None - self.package_output = None - self.packages = [] - self.publish_to = {} - self.raw_package_path = None - self.run_long_tests = False - self.sandbox_download_protocols = [] - self.upload = False - self.wheel_access_key_path = None - self.wheel_secret_key_path = None - - @staticmethod - def consumer(): - return [ - core.yarg.ArgConsumer( - names=['--publish-to'], - help='Publish package to the specified dist', - hook=core.yarg.DictPutHook('publish_to'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--build-only'], - hook=core.yarg.SetConstValueHook('build_only', True), - visible=False, - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--change-log'], - help='Change log text or path to the existing changelog file', - hook=core.yarg.SetValueHook('change_log'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--new'], - help='Use new ya package json format', - hook=core.yarg.SetConstValueHook('convert', False), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--old'], - help='Use old ya package json format', - hook=core.yarg.SetConstValueHook('convert', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--tests-data-root'], - help="Custom location for arcadia_tests_data dir, defaults to <source root>/../arcadia_tests_data", - hook=core.yarg.SetValueHook('custom_tests_data_root'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--data-root'], - help="Custom location for data dir, defaults to <source root>/../data", - hook=core.yarg.SetValueHook('custom_data_root'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--artifactory-password-path'], - help='Path to file with artifactory password', - hook=core.yarg.SetValueHook('artifactory_password_path'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.EnvConsumer( - 'YA_ARTIFACTORY_PASSWORD_PATH', - help='Path to file with artifactory password', - hook=core.yarg.SetValueHook('artifactory_password_path'), - ), - core.yarg.ArgConsumer( - names=['--dump-arcadia-inputs'], - help='Only dump inputs, do not build package', - hook=core.yarg.SetValueHook('dump_inputs'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--ignore-fail-tests'], - help='Create package, no matter tests failed or not', - hook=core.yarg.SetConstValueHook('ignore_fail_tests', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--codec'], - help='Codec name for uc compression', - hook=core.yarg.SetValueHook('codec'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--codecs-list'], - help='Show available codecs for --uc', - hook=core.yarg.SetConstValueHook('list_codecs', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - ["-O", "--package-output"], - help="Specifies directory for package output", - hook=core.yarg.SetValueHook('package_output'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.FreeArgConsumer( - help='Package description file name(s)', - hook=core.yarg.SetValueHook(name='packages'), - ), - core.yarg.ArgConsumer( - ['--sandbox-download-protocol'], - help='Sandbox download protocols comma-separated (default: http,http_tgz)', - hook=core.yarg.SetValueHook( - 'sandbox_download_protocols', transform=lambda val: [_f for _f in val.split(",") if _f] - ), - visible=False, - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--wheel-repo-access-key'], - help='Path to access key for wheel repository', - hook=core.yarg.SetValueHook('wheel_access_key_path'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=PYTHON_WHEEL_SUBGROUP, - ), - core.yarg.EnvConsumer( - 'YA_WHEEL_REPO_ACCESS_KEY_PATH', - help='Path to access key for wheel repository', - hook=core.yarg.SetValueHook('wheel_access_key_path'), - ), - core.yarg.ArgConsumer( - names=['--wheel-repo-secret-key'], - help='Path to secret key for wheel repository', - hook=core.yarg.SetValueHook('wheel_secret_key_path'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=PYTHON_WHEEL_SUBGROUP, - ), - core.yarg.EnvConsumer( - 'YA_WHEEL_SECRET_KEY_PATH', - help='Path to secret key for wheel repository', - hook=core.yarg.SetValueHook('wheel_secret_key_path'), - ), - core.yarg.ArgConsumer( - names=['--raw-package-path'], - help="Custom path for raw-package (implies --raw-package)", - hook=core.yarg.SetValueHook('raw_package_path'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--no-cleanup'], - help='Do not clean the temporary directory', - hook=core.yarg.SetConstValueHook('cleanup', False), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--build-debian-scripts'], - hook=core.yarg.SetConstValueHook('build_debian_scripts', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--debian-distribution'], - help='Debian distribution', - hook=core.yarg.SetValueHook('debian_distribution'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.EnvConsumer( - 'YA_DEBIAN_UPLOAD_TOKEN', - help='Iam token or path to iam token for nebiuscloud debian repository', - hook=core.yarg.SetValueHook('debian_upload_token'), - ), - core.yarg.ArgConsumer( - names=['--docker-push'], - help='Push docker image to registry', - hook=core.yarg.SetConstValueHook('docker_push_image', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-no-cache'], - help='Disable docker cache', - hook=core.yarg.SetConstValueHook('docker_no_cache', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ConfigConsumer("docker_no_cache"), - core.yarg.ArgConsumer( - names=['--dump-build-targets'], - hook=core.yarg.SetValueHook('dump_build_targets'), - visible=False, - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-use-remote-cache'], - help='Use image from registry as cache source', - hook=core.yarg.SetConstValueHook('docker_use_remote_cache', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-remote-image-version'], - help='Specify image version to be used as cache source', - hook=core.yarg.SetValueHook('docker_remote_image_version'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--nanny-release'], - help='Notify nanny about new release', - hook=core.yarg.SetValueHook( - 'nanny_release', transform=lambda s: s.upper(), values=const.NANNY_RELEASE_TYPES - ), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - ['--upload'], - help='Upload created package to sandbox', - hook=core.yarg.SetConstValueHook('upload', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - ] - - def postprocess(self): - if self.convert is not None: - logger.warning("Package format will be detected automatically, no need to use --new and --old") - if self.nanny_release and not self.docker_push_image: - raise core.yarg.ArgsValidatingException("Using --nanny-release without --docker-push is pointless") - - def postprocess2(self, params): - if params.raw_package_path and not params.raw_package: - params.raw_package = True - # old params compatibility - if getattr(params, 'run_long_tests', False): - params.run_tests = test_opts.RunTestOptions.RunAllTests - - -class PackageCustomizableOptions(core.yarg.Options): - """ - Don't add parameters here by default, otherwise user could use them in package.json. - For more info see https://docs.yandex-team.ru/ya-make/usage/ya_package/json#params - """ - - deb_compression_levels = collections.OrderedDict( - sorted( - { - 'none': 0, - 'low': 3, - 'medium': 6, - 'high': 9, - }.items(), - key=lambda i: i[1], - ) - ) - - def __init__(self): - self.arch_all = False - self.artifactory = None - self.compress_archive = True - self.compression_filter = None - self.compression_level = None - self.create_dbg = False - self.custom_version = None - self.debian_arch = None - self.debian_compression_level = None - self.debian_compression_type = 'gzip' - self.docker_add_host = [] - self.docker_build_arg = {} - self.docker_build_network = None - self.docker_platform = None - self.docker_registry = "registry.yandex.net" - self.docker_repository = "" - self.docker_save_image = False - self.docker_secrets = [] - self.docker_target = None - self.dupload_max_attempts = 1 - self.dupload_no_mail = False - self.ensure_package_published = False - self.force_dupload = False - self.format = None - self.full_strip = False - self.key = None - self.overwrite_read_only_files = False - self.raw_package = False - self.resource_attrs = {} - self.resource_type = "YA_PACKAGE" - self.sandbox_task_id = 0 - self.sign = True - self.sloppy_deb = False - self.store_debian = True - self.strip = False - self.wheel_platform = "" - self.wheel_python3 = False - - @staticmethod - def consumer(): - return [ - core.yarg.ArgConsumer( - names=['--strip'], - help='Strip binaries (only debug symbols: "strip -g")', - hook=core.yarg.SetConstValueHook('strip', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--full-strip'], - help='Strip binaries', - hook=core.yarg.SetConstValueHook('full_strip', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--set-sandbox-task-id'], - visible=False, - help='Use the provided task id for the package version if needed', - hook=core.yarg.SetValueHook('sandbox_task_id', int), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--wheel-platform'], - visible=True, - help='Set wheel package platform', - hook=core.yarg.SetValueHook('wheel_platform'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=PYTHON_WHEEL_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--key'], - help='The key to use for signing', - hook=core.yarg.SetValueHook('key'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--debian'], - help='Build debian package', - hook=core.yarg.SetConstValueHook('format', const.PackageFormat.DEBIAN), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--tar'], - help='Build tarball package', - hook=core.yarg.SetConstValueHook('format', const.PackageFormat.TAR), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--no-compression'], - help="Don't compress tar archive (for --tar only)", - hook=core.yarg.SetConstValueHook('compress_archive', False), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--create-dbg'], - help='Create separate package with debug info (works only in case of --strip or --full-strip)', - hook=core.yarg.SetConstValueHook('create_dbg', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - ["--compression-filter"], - help="Specifies compression filter (gzip/zstd)", - hook=core.yarg.SetValueHook('compression_filter'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - ["--compression-level"], - help="Specifies compression level (0-9 for gzip [6 is default], 0-22 for zstd [3 is default])", - hook=core.yarg.SetValueHook('compression_level', transform=lambda s: int(s)), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker'], - help='Build docker', - hook=core.yarg.SetConstValueHook('format', const.PackageFormat.DOCKER), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--rpm'], - help='Build rpm package', - hook=core.yarg.SetConstValueHook('format', const.PackageFormat.RPM), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=RPM_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--aar'], - help='Build aar package', - hook=core.yarg.SetConstValueHook('format', const.PackageFormat.AAR), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=AAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--npm'], - help='Build npm package', - hook=core.yarg.SetConstValueHook('format', const.PackageFormat.NPM), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=NPM_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--wheel'], - help='Build wheel package', - hook=core.yarg.SetConstValueHook('format', const.PackageFormat.WHEEL), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=PYTHON_WHEEL_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--wheel-python3'], - help='use python3 when building wheel package', - hook=core.yarg.SetConstValueHook('wheel_python3', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=PYTHON_WHEEL_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--artifactory'], - help='Build package and upload it to artifactory', - hook=core.yarg.SetConstValueHook("artifactory", True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-add-host'], - help='Docker --add-host', - hook=core.yarg.SetAppendHook('docker_add_host'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-secret'], - help='Same as Docker --secret. You can pass few secrets at the same time', - hook=core.yarg.SetAppendHook('docker_secrets'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-registry'], - help='Docker registry', - hook=core.yarg.SetValueHook('docker_registry'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-repository'], - help='Specify private repository', - hook=core.yarg.SetValueHook('docker_repository'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-save-image'], - help='Save docker image to archive', - hook=core.yarg.SetConstValueHook('docker_save_image', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-network'], - help='--network parameter for `docker build` command', - hook=core.yarg.SetValueHook('docker_build_network'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-platform'], - help='Specify platform for docker build (require buildx)', - hook=core.yarg.SetValueHook('docker_platform'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-build-arg'], - help='--build-arg parameter for `docker build` command, set it in the <key>=<value> form', - hook=core.yarg.DictPutHook('docker_build_arg'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--docker-target'], - help='Specifying target build stage (--target)', - hook=core.yarg.SetValueHook('docker_target'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DOCKER_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--raw-package'], - help="Used with --tar to get package content without tarring", - hook=core.yarg.SetConstValueHook('raw_package', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=TAR_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--sloppy-and-fast-debian'], - help="Fewer checks and no compression when building debian package", - hook=core.yarg.SetConstValueHook('sloppy_deb', True), - visible=False, - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--not-sign-debian'], - help='Do not sign debian package', - hook=core.yarg.SetConstValueHook('sign', False), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--custom-version'], - help='Custom package version', - hook=core.yarg.SetValueHook('custom_version'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--debian-arch'], - help='Debian arch (passed to debuild as `-a`)', - hook=core.yarg.SetValueHook('debian_arch'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--arch-all'], - help='Use "Architecture: all" in debian', - hook=core.yarg.SetConstValueHook('arch_all', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--force-dupload'], - help='dupload --force', - hook=core.yarg.SetConstValueHook('force_dupload', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['-z', '--debian-compression'], - help="deb-file compresson level ({})".format( - ", ".join(list(PackageCustomizableOptions.deb_compression_levels.keys())) - ), - hook=core.yarg.SetValueHook( - 'debian_compression_level', values=list(PackageCustomizableOptions.deb_compression_levels.keys()) - ), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['-Z', '--debian-compression-type'], - help="deb-file compression type used when building deb-file (allowed types: {}, gzip (default), xz, bzip2, lzma, none)".format( - const.DEBIAN_HOST_DEFAULT_COMPRESSION_LEVEL - ), - hook=core.yarg.SetValueHook('debian_compression_type'), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--dont-store-debian'], - help="Save debian package in a separate archive", - hook=core.yarg.SetConstValueHook('store_debian', False), - visible=False, - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - ['--upload-resource-type'], - help='Created resource type', - hook=core.yarg.SetValueHook('resource_type'), - group=build.build_opts.SANDBOX_UPLOAD_OPT_GROUP, - ), - core.yarg.ArgConsumer( - ['--upload-resource-attr'], - help='Resource attr, set it in the <name>=<value> form', - hook=core.yarg.DictPutHook(name='resource_attrs'), - group=build.build_opts.SANDBOX_UPLOAD_OPT_GROUP, - ), - core.yarg.ArgConsumer( - names=['--dupload-max-attempts'], - help='How many times try to run dupload if it fails', - hook=core.yarg.SetValueHook('dupload_max_attempts', int), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--dupload-no-mail'], - help='dupload --no-mail', - hook=core.yarg.SetConstValueHook('dupload_no_mail', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=DEB_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--overwrite-read-only-files'], - help='Overwrite read-only files in package', - hook=core.yarg.SetConstValueHook('overwrite_read_only_files', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - core.yarg.ArgConsumer( - names=['--ensure-package-published'], - help='Ensure that package is available in the repository', - hook=core.yarg.SetConstValueHook('ensure_package_published', True), - group=core.yarg.PACKAGE_OPT_GROUP, - subgroup=COMMON_SUBGROUP, - ), - ] - - def postprocess(self): - if self.debian_compression_level is not None: - self.debian_compression_level = self.deb_compression_levels[self.debian_compression_level] - if self.create_dbg: - if not self.full_strip: - self.strip = True - if self.compression_filter not in (None, 'gzip', 'zstd'): - raise core.yarg.ArgsValidatingException( - "Using unsupported compression filter: {}".format(self.compression_filter) - ) - - -class InterimOptions(core.yarg.Options): - Visible = False - - def __init__(self): - self.verify_patterns_usage = True - - # All this options - # - !! should never be available in YA_PACKAGE sandbox task !! - # - will be removed when work is done - def consumer(self): - return [ - core.yarg.ArgConsumer( - names=['--fixme-CHEMODAN-80080'], - help='See CHEMODAN-80080 and DEVTOOLSSUPPORT-12411 for more info', - hook=core.yarg.SetConstValueHook('verify_patterns_usage', False), - visible=self.Visible, - ), - ] diff --git a/devtools/ya/handlers/package/opts/ya.make b/devtools/ya/handlers/package/opts/ya.make deleted file mode 100644 index f13c20f4b58..00000000000 --- a/devtools/ya/handlers/package/opts/ya.make +++ /dev/null @@ -1,19 +0,0 @@ -PY23_LIBRARY() - -PY_SRCS( - __init__.py -) - -PEERDIR( - devtools/ya/build/build_opts - devtools/ya/core - devtools/ya/core/yarg - devtools/ya/exts - devtools/ya/package/const - devtools/ya/test/const - devtools/ya/test/opts -) - -STYLE_PYTHON() - -END() diff --git a/devtools/ya/handlers/package/ya.make b/devtools/ya/handlers/package/ya.make deleted file mode 100644 index 98c8a47dd99..00000000000 --- a/devtools/ya/handlers/package/ya.make +++ /dev/null @@ -1,27 +0,0 @@ -PY23_LIBRARY() - -PY_SRCS( - NAMESPACE handlers.package - __init__.py -) - -PEERDIR( - contrib/python/pathlib2 - devtools/ya/app - devtools/ya/build - devtools/ya/build/build_opts - devtools/ya/core - devtools/ya/core/yarg - devtools/ya/handlers/package/opts - devtools/ya/package - devtools/ya/test/opts -) - -STYLE_PYTHON() - -END() - -RECURSE_FOR_TESTS( - bin - opts -) diff --git a/devtools/ya/handlers/test/__init__.py b/devtools/ya/handlers/test/__init__.py deleted file mode 100644 index 2b3b2e4de48..00000000000 --- a/devtools/ya/handlers/test/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -import app - -from build.build_handler import do_ya_make -from build.build_opts import ya_make_options - -import core.yarg - - -class YaTestYaHandler(core.yarg.OptsHandler): - description = 'Build and run all tests\n[[imp]]ya test[[rst]] is alias for [[imp]]ya make -A[[rst]]' - - def __init__(self): - core.yarg.OptsHandler.__init__( - self, - action=app.execute(action=do_ya_make), - examples=[ - core.yarg.UsageExample( - '{prefix}', - 'Build and run all tests', - good_looking=101, - ), - core.yarg.UsageExample( - '{prefix} -t', - 'Build and run small tests only', - good_looking=102, - ), - core.yarg.UsageExample( - '{prefix} -tt', - 'Build and run small and medium tests', - good_looking=103, - ), - ], - description=self.description, - opts=ya_make_options( - free_build_targets=True, - is_ya_test=True, - strip_idle_build_results=True, - ), - visible=True, - ) diff --git a/devtools/ya/handlers/test/ya.make b/devtools/ya/handlers/test/ya.make deleted file mode 100644 index 70e9dee8a19..00000000000 --- a/devtools/ya/handlers/test/ya.make +++ /dev/null @@ -1,21 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.test - __init__.py -) - -PEERDIR( - devtools/ya/app - devtools/ya/build - devtools/ya/build/build_opts -) - -END() - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/tool/__init__.py b/devtools/ya/handlers/tool/__init__.py deleted file mode 100644 index b4ac4ccdca9..00000000000 --- a/devtools/ya/handlers/tool/__init__.py +++ /dev/null @@ -1,281 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -import os -import sys -import logging - -from core.common_opts import CrossCompilationOptions -from core.yarg import ( - ArgConsumer, - CompositeHandler, - EnvConsumer, - SetConstValueHook, - SetValueHook, - Options, - OptsHandler, - FreeArgConsumer, - ConfigConsumer, - ExtendHook, - ShowHelpException, - SetAppendHook, - BaseHook, -) - -import app - -from build.build_opts import CustomFetcherOptions, SandboxAuthOptions, ToolsOptions -from core.yarg.groups import PRINT_CONTROL_GROUP -from core.yarg.help_level import HelpLevel -from yalibrary.tools import environ, param, resource_id, task_id, tool, tools, toolchain_root, toolchain_sys_libs -from yalibrary.toolscache import lock_resource -from yalibrary.platform_matcher import is_darwin_arm64 -import core.config -import core.respawn -import exts.process -import exts.windows - -logger = logging.getLogger(__name__) - - -class ToolYaHandler(CompositeHandler): - description = 'Execute specific tool' - - @staticmethod - def common_download_options(): - return [SandboxAuthOptions(), CustomFetcherOptions(), ToolsOptions()] - - def __init__(self): - CompositeHandler.__init__( - self, - description=self.description, - examples=[ - core.yarg.UsageExample('{prefix} --ya-help', 'Print yatool specific options', good_looking=20), - core.yarg.UsageExample('{prefix} --print-path', 'Print path to tool executable file', good_looking=10), - core.yarg.UsageExample( - '{prefix} --force-update', - 'Check tool for updates before the update interval elapses', - good_looking=10, - ), - ], - ) - for x in tools(): - self[x.name] = OptsHandler( - action=app.execute(action=do_tool, respawn=app.RespawnType.OPTIONAL), - description=x.description, - visible=x.visible, - opts=[ToolOptions(x.name)] + self.common_download_options(), - unknown_args_as_free=True, - ) - - -class DummyHook(BaseHook): - def __call__(self, to, *args): - # type: ("Options", tp.Optional[tp.Any]) -> None - pass - - @staticmethod - def need_value(): - return False - - -class ToolOptions(Options): - def __init__(self, tool): - Options.__init__(self) - self.tool = tool - self.print_path = None - self.print_toolchain_path = None - self.print_toolchain_sys_libs = None - self.toolchain = None - self.param = None - self.platform = None - self.target_platforms = [] - self.need_task_id = None - self.need_resource_id = None - self.show_help = False - self.tail_args = [] - self.host_platform = None - self.hide_arm64_host_warning = False - self.force_update = False - self.force_refetch = False - - @staticmethod - def consumer(): - return [ - ArgConsumer( - ['--print-path'], - help='Only print path to tool, do not execute', - hook=SetConstValueHook('print_path', True), - ), - ArgConsumer( - ['--print-toolchain-path'], - help='Print path to toolchain root', - hook=SetConstValueHook('print_toolchain_path', True), - ), - ArgConsumer( - ['--print-toolchain-sys-libs'], - help='Print pathes to toolchsin system libraries', - hook=SetConstValueHook('print_toolchain_sys_libs', True), - ), - ArgConsumer(['--platform'], help="Set specific platform", hook=SetValueHook('platform')), - ArgConsumer(['--host-platform'], help="Set host platform", hook=SetValueHook('host_platform')), - EnvConsumer('YA_TOOL_HOST_PLATFORM', hook=SetValueHook('host_platform')), - ArgConsumer(['--toolchain'], help="Specify toolchain", hook=SetValueHook('toolchain')), - ArgConsumer(['--get-param'], help="Get specified param", hook=SetValueHook('param')), - ArgConsumer( - ['--get-resource-id'], - help="Get resource id for specific platform (the platform should be specified)", - hook=SetConstValueHook('need_resource_id', True), - ), - ArgConsumer(['--get-task-id'], help="Get task id", hook=SetConstValueHook('need_task_id', True)), - ArgConsumer(['--ya-help'], help="Show help", hook=SetConstValueHook('show_help', True)), - ArgConsumer( - ['--target-platform'], - help='Target platform', - hook=SetAppendHook('target_platforms', values=CrossCompilationOptions.generate_target_platforms_cxx()), - ), - ArgConsumer( - ['--hide-arm64-host-warning'], - help='Hide MacOS arm64 host warning', - hook=SetConstValueHook('hide_arm64_host_warning', True), - group=PRINT_CONTROL_GROUP, - visible=HelpLevel.EXPERT if is_darwin_arm64() else False, - ), - EnvConsumer('YA_TOOL_HIDE_ARM64_HOST_WARNING', hook=SetConstValueHook('hide_arm64_host_warning', True)), - ConfigConsumer('hide_arm64_host_warning'), - ArgConsumer( - ['--force-update'], - help='Check tool for updates before the update interval elapses', - hook=SetConstValueHook('force_update', True), - ), - ArgConsumer(['--force-refetch'], help='Refetch toolchain', hook=SetConstValueHook('force_refetch', True)), - ArgConsumer(['--print-fastpath-error'], help='Print fast path failure error', hook=DummyHook()), - FreeArgConsumer(help='arg', hook=ExtendHook(name='tail_args')), - ] - - def postprocess(self): - if self.show_help: - raise ShowHelpException() - if self.toolchain and self.target_platforms: - raise core.yarg.ArgsValidatingException("Do not use --toolchain and --target-platform args together") - if self.force_update: - os.environ['YA_TOOL_FORCE_UPDATE'] = "1" - - -def _replace(s, transformations): - for k, v in transformations.items(): - s = s.replace('$({})'.format(k), v) - return s - - -def _useful_env_vars(): - return {'YA_TOOL': sys.argv[0]} - - -def do_tool(params): - tool_name = params.tool - extra_args = params.tail_args - target_platform = params.target_platforms - host_platform = params.host_platform - if target_platform: - if len(target_platform) > 1: - raise Exception('Multiple target platforms are not supported by this code for now') - target_platform = target_platform[0] - else: - target_platform = None - - if is_darwin_arm64() and not host_platform: - host_platform = 'darwin' - if not params.hide_arm64_host_warning: - try: - import app_ctx - - app_ctx.display.emit_message("You use x86_64 version of selected tool.") - except Exception as e: - logger.error("Can't print arm64 warning message: {}".format(e)) - - tool_path = tool( - tool_name, - params.toolchain, - target_platform=target_platform, - for_platform=host_platform, - force_refetch=params.force_refetch, - ) - if exts.windows.on_win() and not tool_path.endswith('.exe'): # XXX: hack. Think about ya.conf.json format - logger.debug('Rename tool for win: %s', tool_path) - tool_path += '.exe' - - lock_result = False - for_platform = params.platform or params.host_platform or None - - if params.need_task_id: - tid = task_id(tool_name, params.toolchain) - if tid is not None: - print(tid) - else: - raise Exception("Tool '{}' has no task id".format(tool_name)) - elif params.need_resource_id: - print(resource_id(tool_name, params.toolchain, for_platform)) - elif params.param: - print(param(tool_name, params.toolchain, params.param)) - elif params.print_toolchain_path: - print(toolchain_root(tool_name, params.toolchain, for_platform)) - lock_result = True - elif params.print_toolchain_sys_libs: - print(toolchain_sys_libs(tool_name, params.toolchain, for_platform)) - lock_result = True - elif params.print_path: - print(tool_path) - lock_result = True - elif os.path.isfile(tool_path): - env = core.respawn.filter_env(os.environ.copy()) - - # Remove environment variables set by 'ya' wrapper. - # They are actually one-time ya-bin parameters rather than inheritable environment - # for all descendant processes. - for key in ('YA_SOURCE_ROOT', 'YA_PYVER_REQUIRE', 'YA_PYVER_SET_FORCED'): - env.pop(key, None) - - env.update(_useful_env_vars()) - for key, value in environ(tool_name, params.toolchain).items(): - env[key] = _replace( - os.pathsep.join(value), {'ROOT': toolchain_root(tool_name, params.toolchain, for_platform)} - ) - if tool_name == 'gdb': - # gdb does not fit in 8 MB stack with large cores (DEVTOOLS-5040). - try: - import resource as r - except ImportError: - pass - else: - soft, hard = r.getrlimit(r.RLIMIT_STACK) - new = 128 << 20 - logger.debug("Limit info: soft=%d hard=%d new=%d", soft, hard, new) - if hard != r.RLIM_INFINITY: - new = min(new, hard) - logger.debug("Limit info: new=%d", new) - if new > soft: - logger.debug("Limit info: setting new limits=(%d, %d)", new, hard) - try: - r.setrlimit(r.RLIMIT_STACK, (new, hard)) - except ValueError as e: - logger.error("Failure while setting RLIMIT_STACK ({}, {}), {}".format(new, hard, e)) - logger.exception("While setting RLIMIT_STACK") - arc_root = core.config.find_root(fail_on_error=False) - if arc_root is not None: - logger.debug('Arcadia root found: %s', arc_root) - extra_args = ['-ex', 'set substitute-path /-S/ {}/'.format(arc_root)] + extra_args - extra_args = ['-ex', 'set filename-display absolute'] + extra_args - if ( - tool_name == 'arc' - and params.username not in {'sandbox', 'root'} - and os.getenv('YA_ALLOW_TOOL_ARC', 'no') != 'yes' - ): - message = ( - 'Please, use natively installed arc, install guide:' - ' https://docs.yandex-team.ru/devtools/intro/quick-start-guide#arc-setup' - ) - raise core.yarg.ArgsValidatingException(message) - exts.process.execve(tool_path, extra_args, env=env) - - if lock_result: - lock_resource(toolchain_root(tool_name, params.toolchain, for_platform)) diff --git a/devtools/ya/handlers/tool/ya.make b/devtools/ya/handlers/tool/ya.make deleted file mode 100644 index 6021f3ea29e..00000000000 --- a/devtools/ya/handlers/tool/ya.make +++ /dev/null @@ -1,33 +0,0 @@ -PY23_LIBRARY() - -STYLE_PYTHON() - -PY_SRCS( - NAMESPACE handlers.tool - __init__.py -) - -PEERDIR( - devtools/ya/app - devtools/ya/build/build_opts - devtools/ya/core - devtools/ya/core/config - devtools/ya/core/respawn - devtools/ya/core/yarg - devtools/ya/exts - devtools/ya/yalibrary/platform_matcher - devtools/ya/yalibrary/tools - devtools/ya/yalibrary/toolscache - library/python/windows -) - -END() - -RECURSE( - tests -) - -RECURSE_FOR_TESTS( - bin - tests -) diff --git a/devtools/ya/handlers/ya.make b/devtools/ya/handlers/ya.make deleted file mode 100644 index d88914356d8..00000000000 --- a/devtools/ya/handlers/ya.make +++ /dev/null @@ -1,151 +0,0 @@ -PY23_LIBRARY(ya-lib) - -SRCDIR(devtools/ya) - -PEERDIR( - devtools/ya/handlers/make - devtools/ya/handlers/package - devtools/ya/handlers/test - devtools/ya/handlers/tool - devtools/ya/handlers/ide - devtools/ya/handlers/dump - devtools/ya/handlers/gc - devtools/ya/handlers/gen_config - devtools/ya/handlers/maven_import - devtools/ya/handlers/java - # devtools/ya/handlers/analyze_make -) - -IF (NOT YA_OPENSOURCE) - PEERDIR( - devtools/ya/handlers/__trace__ - devtools/ya/handlers/addremove - devtools/ya/handlers/analyze_make - devtools/ya/handlers/buf - devtools/ya/handlers/check - devtools/ya/handlers/clang_tidy - devtools/ya/handlers/clone - devtools/ya/handlers/completion - devtools/ya/handlers/dctl - devtools/ya/handlers/download - devtools/ya/handlers/exec - devtools/ya/handlers/fetch - devtools/ya/handlers/fix_includes - devtools/ya/handlers/krevedko - devtools/ya/handlers/notify - devtools/ya/handlers/paste - devtools/ya/handlers/pr - devtools/ya/handlers/profile - devtools/ya/handlers/project - devtools/ya/handlers/py - devtools/ya/handlers/remote_gdb - devtools/ya/handlers/repo_check - devtools/ya/handlers/shell - devtools/ya/handlers/shelve - devtools/ya/handlers/stat - devtools/ya/handlers/style - devtools/ya/handlers/svn - devtools/ya/handlers/unshelve - devtools/ya/handlers/upload - devtools/ya/handlers/vmctl - devtools/ya/handlers/webide - devtools/ya/handlers/whoami - devtools/ya/handlers/wine - devtools/ya/handlers/yav - - devtools/ya/handlers/py23migration/_ya0bin2 - devtools/ya/handlers/py23migration/_ya0bin3 - devtools/ya/handlers/py23migration/_ya2bin0 - devtools/ya/handlers/py23migration/_ya2bin2 - devtools/ya/handlers/py23migration/_ya3bin0 - devtools/ya/handlers/py23migration/_ya3bin3 - ) - IF (PYTHON3) - PEERDIR( - devtools/ya/handlers/py23migration/py23_utils - devtools/ya/handlers/vim - devtools/ya/handlers/curl - devtools/ya/handlers/neovim - devtools/ya/handlers/gdb - devtools/ya/handlers/emacs - devtools/ya/handlers/grep - devtools/ya/handlers/jstyle - devtools/ya/handlers/nile - devtools/ya/handlers/sed - devtools/ya/handlers/ydb - devtools/ya/handlers/yql - ) - ENDIF() -ENDIF() -END() - -RECURSE( - __trace__ - addremove - analyze_make - autocheck - buf - check - clang_tidy - clone - completion - curl - dctl - download - dump - emacs - exec - fetch - fix_includes - gc - gdb - gen_config - grep - ide - java - jstyle - krevedko - make - maven_import - neovim - nile - notify - package - paste - pr - profile - project - py - remote_gdb - repo_check - sed - shell - shelve - stat - style - svn - test - tool - unshelve - upload - vim - vmctl - webide - whoami - wine - yav - ydb - yql -) - -IF (NOT OPENSOURCE) - RECURSE( - py23migration/_ya0bin2 - py23migration/_ya0bin3 - py23migration/_ya2bin0 - py23migration/_ya3bin0 - py23migration/_ya3bin3 - py23migration/_ya2bin2 - py23migration/py23_utils - ) -ENDIF() diff --git a/devtools/ya/opensource/ya.conf b/devtools/ya/opensource/ya.conf deleted file mode 100644 index fb140d072ad..00000000000 --- a/devtools/ya/opensource/ya.conf +++ /dev/null @@ -1,92 +0,0 @@ -# Please keep this in sync with arcadia/ya.conf - -build_cache = true -build_cache_conf = ['cas_logging=true', 'graph_info=true'] -build_cache_master = true -cache_codec = '' -cache_size = 150374182400 -content_uids = true -dir_outputs = true -dir_outputs_test_mode = true -dump_debug_enabled = true -fail_maven_export_with_tests = true -incremental_build_dirs_cleanup = true -oauth_exchange_ssh_keys = true -remove_implicit_data_path = true -remove_result_node = true -tools_cache = true -tools_cache_master = true -use_atd_revisions_info = true -use_jstyle_server = true -use_command_file_in_testtool = true - -[test_tool3_handlers] -build_clang_coverage_report = true -build_go_coverage_report = true -build_python_coverage_report = false -build_sancov_coverage_report = true -build_ts_coverage_report = true -canonization_result_node = true -canonize = true -check_external = true -check_mds = true -check_resource = true -checkout = true -cov_merge_vfs = false -create_allure_report = true -download = true -list_result_node = true -list_tests = true -merge_coverage_inplace = true -merge_python_coverage = true -minimize_fuzz_corpus = true -populate_token_to_sandbox_vault = true -resolve_clang_coverage = true -resolve_go_coverage = true -resolve_java_coverage = true -resolve_python_coverage = false -resolve_sancov_coverage = true -resolve_ts_coverage = true -result_node = true -results_accumulator = true -results_merger = true -run_boost_test = true -run_check = true -run_clang_tidy = true -run_classpath_clash = true -run_coverage_extractor = true -run_custom_lint = true -run_diff_test = false -run_eslint = true -run_exectest = true -run_fuzz = true -run_fuzz_result_node = true -run_g_benchmark = true -run_go_fmt = true -run_go_test = true -run_go_vet = true -run_gtest = true -run_hermione = true -run_javastyle = true -run_jest = true -run_ktlint_test = true -run_pyimports = true -run_skipped_test = true -run_test = true -run_ut = true -run_y_benchmark = true -sandbox_run_test = true -unify_clang_coverage = true -upload = false -upload_coverage = false -ytexec_run_test = true - -# ===== opensource only table params ===== - -[host_platform_flags] -OPENSOURCE = "yes" -USE_PREBUILT_TOOLS = "no" - -[flags] -OPENSOURCE = "yes" -USE_PREBUILT_TOOLS = "no" |
