diff options
4 files changed, 19 insertions, 239 deletions
diff --git a/ydb/library/yql/cfg/tests/gateways-experimental.conf b/ydb/library/yql/cfg/tests/gateways-experimental.conf new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/ydb/library/yql/cfg/tests/gateways-experimental.conf diff --git a/ydb/library/yql/cfg/tests/gateways-forceblocks.conf b/ydb/library/yql/cfg/tests/gateways-forceblocks.conf index 83690243dfa..53540ce3aa1 100644 --- a/ydb/library/yql/cfg/tests/gateways-forceblocks.conf +++ b/ydb/library/yql/cfg/tests/gateways-forceblocks.conf @@ -1,234 +1,5 @@ -Yt { - GatewayThreads: 0 - YtLogLevel: YL_INFO - - DefaultSettings { - Name: "InferSchemaMode" - Value: "rpc" - } - - DefaultSettings { - Name: "DQRPCReaderInflight" - Value: "10" - } - - DefaultSettings { - Name: "UseRPCReaderInDq" - Value: "true" - } - - DefaultSettings { - Name: "DefaultMaxJobFails" - Value: "1" - } - - DefaultSettings { - Name: "ReleaseTempData" - Value: "immediate" - } - - DefaultSettings { - Name: "NativeYtTypeCompatibility" - Value: "all" - } - - DefaultSettings { - Name: "UseYqlRowSpecCompactForm" - Value: "true" - } - - DefaultSettings { - Name: "UseNewPredicateExtraction" - Value: "true" - } - - DefaultSettings { - Name: "PruneKeyFilterLambda" - Value: "true" - } - - DefaultSettings { - Name: "_UseKeyBoundApi" - Value: "true" - } - - DefaultSettings { - Name: "JoinCommonUseMapMultiOut" - Value: "true" - } - - DefaultSettings { - Name: "UseAggPhases" - Value: "true" - } - - DefaultSettings { - Name: "_EnableWriteReorder" - Value: "true" - } - - DefaultSettings { - Name: "_EnableYtPartitioning" - Value: "true" - } - - DefaultSettings { - Name: "TableContentLocalExecution" - Value: "true" - } - - RemoteFilePatterns { - Pattern: "yt://([a-zA-Z0-9\\-_]+)/([^&@?]+)$" - Cluster: "$1" - Path: "$2" - } - - RemoteFilePatterns { - Pattern: "https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&path=//([^&]+)$" - Cluster: "$2" - Path: "$3" - } - - RemoteFilePatterns { - Pattern: "https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?path=//([^&]+)$" - Cluster: "$2" - Path: "$3" - } - - RemoteFilePatterns { - Pattern: "https?://([a-zA-Z0-9\\-_]+)\\.yt\\.yandex(\\.net|-team\\.ru)/api/v(2/download|3/read_file)\\?(|.+&)path=//([^&]+)($|&.*)" - Cluster: "$1" - Path: "$5" - } -} - -SqlCore { - TranslationFlags: ["FlexibleTypes", "DisableAnsiOptionalAs", "EmitAggApply"] -} - YqlCore { Flags { - Name: "DisableYsonCastToString" - } - Flags { - Name: "_EnableMatchRecognize" - } - Flags { Name: "UseBlocks" } } - -Dq { - DefaultSettings { - Name: "EnableLLVM" - Value: "true" - } - - DefaultSettings { - Name: "EnableInsert" - Value: "true" - } - - DefaultSettings { - Name: "EnableFullResultWrite" - Value: "true" - } - - DefaultSettings { - Name: "_EnablePrecompute" - Value: "true" - } - - DefaultSettings { - Name: "_LiteralTimeout" - Value: "3600000" - } - - DefaultSettings { - Name: "_TableTimeout" - Value: "3600000" - } - DefaultSettings { - Name: "EnableComputeActor" - Value: "1" - } - DefaultSettings { - Name: "ComputeActorType" - Value: "async" - } - - DefaultSettings { - Name: "UseAggPhases" - Value: "true" - } - - DefaultSettings { - Name: "HashJoinMode" - Value: "grace" - } - - DefaultSettings { - Name: "UseFastPickleTransport" - Value: "true" - } - - DefaultSettings { - Name: "UseOOBTransport" - Value: "true" - } - - DefaultSettings { - Name: "UseFinalizeByKey" - Value: "true" - } - - DefaultSettings { - Name: "EnableDqReplicate" - Value: "true" - } - - DefaultSettings { - Name: "UseWideChannels" - Value: "true" - } -} - -Fs { - CustomSchemes { - Pattern: "sbr:(?://)?(\\d+)" - TargetUrl: "https://proxy.sandbox.yandex-team.ru/$1" - } - CustomSchemes { - # backward compatibility rule - Pattern: "yt://([a-zA-Z0-9\\-_]+)/(.+)@t=([^&=]+)" - TargetUrl: "yt://$1/$2?transaction_id=$3" - } - CustomSchemes { - Pattern: "https?://yt\\.yandex(?:\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&(.+)" - TargetUrl: "yt://$1/?$2" - } - CustomSchemes { - Pattern: "https?://yt\\.yandex(?:\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?(.+)" - TargetUrl: "yt://$1/?$2" - } - CustomSchemes { - Pattern: "https?://([a-zA-Z0-9\\-_]+)\\.yt\\.yandex(?:\\.net|-team\\.ru)/api/v3/read_file\\?(.+)" - TargetUrl: "yt://$1/?$2" - } - CustomSchemes { - Pattern: "https?://a\\.yandex-team\\.ru/(?:arc|svn)/(.+)/arcadia/(.+)\\?rev=r?(\\d+)" - TargetUrl: "arc:/$2?rev=$3&branch=$1" - } - CustomSchemes { - Pattern: "https?://a(?:rcanum)?\\.yandex-team\\.ru/arcadia/(.+)\\?rev=r(\\d+)" - TargetUrl: "arc:/$1?rev=$2" - } - CustomSchemes { - Pattern: "https?://a(?:rcanum)?\\.yandex-team\\.ru/arcadia/(.+)\\?rev=([^r].*)" - TargetUrl: "arc:/$1?hash=$2" - } - CustomSchemes { - Pattern: "arc://(.*)" - TargetUrl: "arc:/$1" - } -} diff --git a/ydb/library/yql/tests/common/test_framework/yql_utils.py b/ydb/library/yql/tests/common/test_framework/yql_utils.py index 78b9d408997..abb6cb583dd 100644 --- a/ydb/library/yql/tests/common/test_framework/yql_utils.py +++ b/ydb/library/yql/tests/common/test_framework/yql_utils.py @@ -10,6 +10,7 @@ import re import tempfile import shutil +from google.protobuf import text_format from collections import namedtuple, defaultdict, OrderedDict from functools import partial import codecs @@ -37,11 +38,6 @@ def get_param(name, default=None): return yatest.common.get_param(name, os.environ.get(name) or default) -def get_gateway_cfg_suffix(): - default_suffix = None - return get_param('gateway_config_suffix', default_suffix) or '' - - def do_custom_query_check(res, sql_query): custom_check = re.search(r"/\* custom check:(.*)\*/", sql_query) if not custom_check: @@ -54,6 +50,11 @@ def do_custom_query_check(res, sql_query): return True +def get_gateway_cfg_suffix(): + default_suffix = None + return get_param('gateway_config_suffix', default_suffix) or '' + + def get_gateway_cfg_filename(): suffix = get_gateway_cfg_suffix() if suffix == '': @@ -62,6 +63,17 @@ def get_gateway_cfg_filename(): return 'gateways-' + suffix + '.conf' +def merge_default_gateway_cfg(cfg_dir, gateway_config): + + with open(yql_source_path(os.path.join(cfg_dir, 'gateways.conf'))) as f: + text_format.Merge(f.read(), gateway_config) + + suffix = get_gateway_cfg_suffix() + if suffix: + with open(yql_source_path(os.path.join(cfg_dir, 'gateways-' + suffix + '.conf'))) as f: + text_format.Merge(f.read(), gateway_config) + + def find_file(path): arcadia_root = '.' while '.arcadia.root' not in os.listdir(arcadia_root): diff --git a/ydb/library/yql/tests/common/test_framework/yqlrun.py b/ydb/library/yql/tests/common/test_framework/yqlrun.py index 7f1fbe54b85..396590435ed 100644 --- a/ydb/library/yql/tests/common/test_framework/yqlrun.py +++ b/ydb/library/yql/tests/common/test_framework/yqlrun.py @@ -61,14 +61,11 @@ class YQLRun(object): if gateway_config is not None: text_format.Merge(gateway_config, self.gateway_config) - if cfg_dir is None: - cfg_dir = 'ydb/library/yql/cfg/tests' - with open(yql_utils.yql_source_path(cfg_dir + '/' + yql_utils.get_gateway_cfg_filename())) as f: - text_format.Merge(f.read(), self.gateway_config) + yql_utils.merge_default_gateway_cfg(cfg_dir or 'ydb/library/yql/cfg/tests', self.gateway_config) self.fs_config = file_storage_pb2.TFileStorageConfig() - with open(yql_utils.yql_source_path(cfg_dir + '/fs.conf')) as f: + with open(yql_utils.yql_source_path(os.path.join(cfg_dir or 'ydb/library/yql/cfg/tests', 'fs.conf'))) as f: text_format.Merge(f.read(), self.fs_config) if fs_config is not None: |