diff options
| author | Ivan Sukhov <evanevannnn@ydb.tech> | 2025-04-24 12:28:48 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-04-24 12:28:48 +0300 |
| commit | 0b821c5a606d88995647de7c17de360498d2bd98 (patch) | |
| tree | 79a4f4ba39dda8ddd1d5622937f2698e421d2c1c | |
| parent | 8ada07de94c7426998981ce765c0e5a378db4cd1 (diff) | |
| download | ydb-0b821c5a606d88995647de7c17de360498d2bd98.tar.gz | |
Solomon in Cloud reading support (#17324)
43 files changed, 414 insertions, 500 deletions
diff --git a/ydb/core/external_sources/external_source_factory.cpp b/ydb/core/external_sources/external_source_factory.cpp index 013f1bc41cf..4e00020d58b 100644 --- a/ydb/core/external_sources/external_source_factory.cpp +++ b/ydb/core/external_sources/external_source_factory.cpp @@ -135,7 +135,7 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri }, { ToString(NYql::EDatabaseType::Solomon), - CreateExternalDataSource(TString{NYql::SolomonProviderName}, {"NONE", "TOKEN"}, {"use_ssl", "grpc_port"}, hostnamePatternsRegEx) + CreateExternalDataSource(TString{NYql::SolomonProviderName}, {"NONE", "TOKEN"}, {"use_ssl", "grpc_location", "project", "cluster"}, hostnamePatternsRegEx) }, { ToString(NYql::EDatabaseType::Iceberg), diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp index f6b151e11a0..8b8232c4c21 100644 --- a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp @@ -88,10 +88,12 @@ TString GetOrEmpty(const NYql::TCreateObjectSettings& container, const TString& "service_name", // oracle "folder_id", // logging "use_ssl", // solomon - "grpc_port", // solomon "reading_mode", // mongodb "unexpected_type_display_mode", // mongodb "unsupported_type_display_mode", // mongodb + "grpc_location", // solomon + "project", // solomon + "cluster" // solomon }; auto& featuresExtractor = settings.GetFeaturesExtractor(); diff --git a/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp b/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp index b3570cf19d4..de9ed2ff259 100644 --- a/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp +++ b/ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp @@ -274,9 +274,10 @@ private: void Fetch() { NActors::TActorSystem* actorSystem = NActors::TActivationContext::ActorSystem(); + std::map<TString, TString> selectors(ReadParams.Source.GetSelectors().begin(), ReadParams.Source.GetSelectors().end()); ListingFuture = SolomonClient - ->ListMetrics(ReadParams.Source.GetSelectors(), PageSize, CurrentPage++) + ->ListMetrics(selectors, PageSize, CurrentPage++) .Subscribe([actorSystem, selfId = SelfId()]( NThreading::TFuture<NSo::TListMetricsResponse> future) -> void { try { diff --git a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp index 953d8bdeb22..6995efe0723 100644 --- a/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp +++ b/ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.cpp @@ -109,7 +109,7 @@ public: SOURCE_LOG_D("Init"); IngressStats.Level = statsLevel; - UseMetricsQueue = ReadParams.Source.HasSelectors(); + UseMetricsQueue = !ReadParams.Source.HasProgram(); auto stringType = ProgramBuilder.NewDataType(NYql::NUdf::TDataType<char*>::Id); DictType = ProgramBuilder.NewDictType(stringType, stringType, false); @@ -219,11 +219,10 @@ public: return; } - auto& metrics = batch.Metrics; + auto& metric = batch.Metric; auto& pointsCount = batch.Response.Result.PointsCount; - ParsePointsCount(metrics, pointsCount); + ParsePointsCount(metric, pointsCount); - SOURCE_LOG_D("HandlePointsCountBatch batch of size " << metrics.size()); TryRequestData(); } @@ -317,7 +316,7 @@ public: auto value = HolderFactory.CreateDirectArrayHolder(ReadParams.Source.GetSystemColumns().size() + ReadParams.Source.GetLabelNames().size(), items); if (auto it = Index.find(SOLOMON_SCHEME_VALUE); it != Index.end()) { - items[it->second] = NUdf::TUnboxedValuePod(values[i]); + items[it->second] = isnan(values[i]) ? NUdf::TUnboxedValuePod() : NUdf::TUnboxedValuePod(values[i]).MakeOptional(); } if (auto it = Index.find(SOLOMON_SCHEME_TYPE); it != Index.end()) { @@ -412,21 +411,17 @@ private: } void RequestPointsCount() { - std::vector<NSo::TMetric> requestMetrics; - requestMetrics.reserve(std::min<ui64>(MetricsPerPointsCountQuery, ListedMetrics.size())); - while (!ListedMetrics.empty() && requestMetrics.size() < MetricsPerPointsCountQuery) { - requestMetrics.push_back(ListedMetrics.back()); - ListedMetrics.pop_back(); - } + NSo::TMetric requestMetric = ListedMetrics.back(); + ListedMetrics.pop_back(); - auto getPointsCountFuture = SolomonClient->GetPointsCount(std::move(requestMetrics)); + auto getPointsCountFuture = SolomonClient->GetPointsCount(requestMetric.Labels); NActors::TActorSystem* actorSystem = NActors::TActivationContext::ActorSystem(); - getPointsCountFuture.Subscribe([actorSystem, metrics = std::move(requestMetrics), selfId = SelfId()]( + getPointsCountFuture.Subscribe([actorSystem, metric = std::move(requestMetric), selfId = SelfId()]( const NThreading::TFuture<NSo::TGetPointsCountResponse>& response) mutable -> void { actorSystem->Send(selfId, new TEvSolomonProvider::TEvPointsCountBatch( - std::move(metrics), + std::move(metric), response.GetValue()) ); }); @@ -454,7 +449,7 @@ private: from = request.From; to = request.To; - getDataFuture = SolomonClient->GetData(metric, from, to); + getDataFuture = SolomonClient->GetData(metric.Labels, from, to); } else { getDataFuture = SolomonClient->GetData( ReadParams.Source.GetProgram(), @@ -476,18 +471,19 @@ private: }); } - void ParsePointsCount(const std::vector<NSo::TMetric>& metrics, const std::vector<ui64>& pointsCount) { + void ParsePointsCount(const NSo::TMetric& metric, ui64 pointsCount) { TInstant from = TInstant::Seconds(ReadParams.Source.GetFrom()); TInstant to = TInstant::Seconds(ReadParams.Source.GetTo()); - for (size_t i = 0; i < metrics.size(); ++i) { - auto& metric = metrics[i]; - auto& count = pointsCount[i]; + auto ranges = SplitTimeIntervalIntoRanges(from, to, pointsCount); - auto ranges = SplitTimeIntervalIntoRanges(from, to, count); - for (const auto& [fromRange, toRange] : ranges) { - MetricsWithTimeRange.emplace_back(metric, fromRange, toRange); - } + if (ranges.empty()) { + CompletedMetricsCount++; + return; + } + + for (const auto& [fromRange, toRange] : ranges) { + MetricsWithTimeRange.emplace_back(metric, fromRange, toRange); } } @@ -535,7 +531,6 @@ private: std::deque<NSo::TTimeseries> MetricsData; size_t ListedMetricsCount = 0; size_t CompletedMetricsCount = 0; - const ui64 MetricsPerPointsCountQuery = 50; const ui64 MaxPointsPerOneMetric = 1000000; TString SourceId; diff --git a/ydb/library/yql/providers/solomon/common/util.cpp b/ydb/library/yql/providers/solomon/common/util.cpp new file mode 100644 index 00000000000..11c9f181b14 --- /dev/null +++ b/ydb/library/yql/providers/solomon/common/util.cpp @@ -0,0 +1,69 @@ +#include "util.h" + +#include <yql/essentials/utils/yql_panic.h> + +#include <util/string/cast.h> +#include <util/string/split.h> +#include <util/string/strip.h> + +namespace NYql::NSo { + +NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) { + switch (clusterType) { + case TSolomonClusterConfig::SCT_SOLOMON: + return NSo::NProto::ESolomonClusterType::CT_SOLOMON; + case TSolomonClusterConfig::SCT_MONITORING: + return NSo::NProto::ESolomonClusterType::CT_MONITORING; + default: + YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterType)); + } +} + +std::map<TString, TString> ExtractSelectorValues(const TString& selectors) { + YQL_ENSURE(selectors.size() >= 2, "Selectors should be at least 2 characters long"); + std::map<TString, TString> result; + + auto selectorValues = StringSplitter(selectors.substr(1, selectors.size() - 2)).Split(',').SkipEmpty().ToList<TString>(); + for (const auto& selectorValue : selectorValues) { + size_t eqPos = selectorValue.find("="); + YQL_ENSURE(eqPos <= selectorValue.size()); + + TString key = StripString(selectorValue.substr(0, eqPos)); + TString value = StripString(selectorValue.substr(eqPos + 1, selectorValue.size() - eqPos - 1)); + YQL_ENSURE(!key.empty()); + YQL_ENSURE(value.size() >= 2); + + result[key] = value.substr(1, value.size() - 2); + } + + return result; +} + +NProto::TDqSolomonSource FillSolomonSource(const TSolomonClusterConfig* config, const TString& project) { + NSo::NProto::TDqSolomonSource source; + + source.SetClusterType(NSo::MapClusterType(config->GetClusterType())); + source.SetUseSsl(config->GetUseSsl()); + + if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { + source.SetProject(config->GetPath().GetProject()); + source.SetCluster(config->GetPath().GetCluster()); + } else { + source.SetProject(project); + } + + source.SetHttpEndpoint(config->GetCluster()); + for (const auto& attr : config->settings()) { + if (attr.name() == "grpc_location"sv) { + source.SetGrpcEndpoint(attr.value()); + } + } + + if (source.GetGrpcEndpoint().empty()) { + source.SetGrpcEndpoint(config->GetCluster()); + } + + return source; +} + +} // namespace NYql::NSo diff --git a/ydb/library/yql/providers/solomon/common/util.h b/ydb/library/yql/providers/solomon/common/util.h new file mode 100644 index 00000000000..b88435f4236 --- /dev/null +++ b/ydb/library/yql/providers/solomon/common/util.h @@ -0,0 +1,14 @@ +#pragma once + +#include <yql/essentials/providers/common/proto/gateways_config.pb.h> +#include <ydb/library/yql/providers/solomon/proto/dq_solomon_shard.pb.h> + +namespace NYql::NSo { + +NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType); + +std::map<TString, TString> ExtractSelectorValues(const TString& selectors); + +NProto::TDqSolomonSource FillSolomonSource(const TSolomonClusterConfig* config, const TString& project); + +} // namespace NYql::NSo diff --git a/ydb/library/yql/providers/solomon/common/ya.make b/ydb/library/yql/providers/solomon/common/ya.make new file mode 100644 index 00000000000..41c6f18964d --- /dev/null +++ b/ydb/library/yql/providers/solomon/common/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + util.cpp +) + +PEERDIR( + ydb/library/yql/providers/solomon/proto + yql/essentials/providers/common/proto + yql/essentials/utils +) + +END() diff --git a/ydb/library/yql/providers/solomon/events/events.h b/ydb/library/yql/providers/solomon/events/events.h index 180494ac211..a1cce12b09e 100644 --- a/ydb/library/yql/providers/solomon/events/events.h +++ b/ydb/library/yql/providers/solomon/events/events.h @@ -71,10 +71,10 @@ struct TEvSolomonProvider { }; struct TEvPointsCountBatch : public NActors::TEventLocal<TEvPointsCountBatch, EvPointsCountBatch> { - std::vector<NSo::TMetric> Metrics; + NSo::TMetric Metric; NSo::TGetPointsCountResponse Response; - TEvPointsCountBatch(std::vector<NSo::TMetric>&& metrics, const NSo::TGetPointsCountResponse& response) - : Metrics(std::move(metrics)) + TEvPointsCountBatch(NSo::TMetric&& metric, const NSo::TGetPointsCountResponse& response) + : Metric(std::move(metric)) , Response(response) {} }; diff --git a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto index 7d7393ca063..660f8f5f87c 100644 --- a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto +++ b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto @@ -49,16 +49,14 @@ message TDownsampling { message TDqSolomonSource { ESolomonClusterType ClusterType = 1; - string Endpoint = 2; + reserved 2; bool UseSsl = 3; string ServiceAccount = 4; TToken Token = 5; string Project = 6; - oneof p { - string Selectors = 7; - string Program = 8; - } + map<string, string> Selectors = 7; + optional string Program = 8; // seconds since Epoch int64 From = 9; int64 To = 10; @@ -67,4 +65,7 @@ message TDqSolomonSource { repeated string LabelNames = 13; map<string, string> Settings = 14; repeated string RequiredLabelNames = 15; + string HttpEndpoint = 16; + string GrpcEndpoint = 17; + optional string Cluster = 18; } diff --git a/ydb/library/yql/providers/solomon/provider/ya.make b/ydb/library/yql/providers/solomon/provider/ya.make index 016593b68c1..c7ea3477141 100644 --- a/ydb/library/yql/providers/solomon/provider/ya.make +++ b/ydb/library/yql/providers/solomon/provider/ya.make @@ -23,6 +23,7 @@ PEERDIR( ydb/library/yql/providers/common/token_accessor/client ydb/library/yql/providers/dq/expr_nodes ydb/library/yql/providers/solomon/actors + ydb/library/yql/providers/solomon/common ydb/library/yql/providers/solomon/expr_nodes ydb/library/yql/providers/solomon/proto ydb/library/yql/providers/solomon/scheme diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp index f04c76b12dc..5f9cbcb5977 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource.cpp @@ -42,9 +42,17 @@ public: cluster.SetToken(token); cluster.SetUseSsl(properties.Value("use_ssl", "true") == "true"sv); - if (auto value = properties.Value("grpc_port", ""); !value.empty()) { + if (properties.Value("project", "") && properties.Value("cluster", "")) { + cluster.SetClusterType(TSolomonClusterConfig::SCT_MONITORING); + cluster.MutablePath()->SetProject(properties.Value("project", "")); + cluster.MutablePath()->SetCluster(properties.Value("cluster", "")); + } else { + cluster.SetClusterType(TSolomonClusterConfig::SCT_SOLOMON); + } + + if (auto value = properties.Value("grpc_location", "")) { auto grpcPort = cluster.MutableSettings()->Add(); - *grpcPort->MutableName() = "grpcPort"; + *grpcPort->MutableName() = "grpc_location"; *grpcPort->MutableValue() = value; } diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp index b4383ec6504..1df9368dffb 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp @@ -4,6 +4,9 @@ #include <yql/essentials/providers/common/provider/yql_provider_names.h> #include <yql/essentials/providers/common/provider/yql_provider.h> +#include <util/string/strip.h> +#include <util/string/split.h> + namespace NYql { using namespace NNodes; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index afe43910ee9..cc7a94f8e6b 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -6,11 +6,11 @@ #include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h> #include <yql/essentials/utils/log/log.h> #include <yql/essentials/providers/common/dq/yql_dq_integration_impl.h> -#include <yql/essentials/providers/common/proto/gateways_config.pb.h> #include <yql/essentials/providers/common/schema/expr/yql_expr_schema.h> #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> #include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h> #include <ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.h> +#include <ydb/library/yql/providers/solomon/common/util.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> #include <ydb/library/yql/providers/solomon/proto/dq_solomon_shard.pb.h> #include <ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h> @@ -37,17 +37,6 @@ bool ExtractSettingValue(const TExprNode& value, TStringBuf settingName, TExprCo return true; } -NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) { - switch (clusterType) { - case TSolomonClusterConfig::SCT_SOLOMON: - return NSo::NProto::ESolomonClusterType::CT_SOLOMON; - case TSolomonClusterConfig::SCT_MONITORING: - return NSo::NProto::ESolomonClusterType::CT_MONITORING; - default: - YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterType)); - } -} - void FillScheme(const TTypeAnnotationNode& itemType, NSo::NProto::TDqSolomonShardScheme& scheme) { int index = 0; for (const TItemExprType* structItem : itemType.Cast<TStructExprType>()->GetItems()) { @@ -290,24 +279,23 @@ public: const auto& cluster = dqSource.DataSource().Cast<TSoDataSource>().Cluster().StringValue(); const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(cluster); YQL_ENSURE(clusterDesc, "Unknown cluster " << cluster); - NSo::NProto::TDqSolomonSource source; - source.SetEndpoint(clusterDesc->GetCluster()); - source.SetProject(settings.Project().StringValue()); - source.SetClusterType(MapClusterType(clusterDesc->GetClusterType())); - source.SetUseSsl(clusterDesc->GetUseSsl()); + NSo::NProto::TDqSolomonSource source = NSo::FillSolomonSource(clusterDesc, settings.Project().StringValue()); + source.SetFrom(TInstant::ParseIso8601(settings.From().StringValue()).Seconds()); source.SetTo(TInstant::ParseIso8601(settings.To().StringValue()).Seconds()); - - auto& sourceSettings = *source.MutableSettings(); - - for (const auto& attr : clusterDesc->settings()) { - sourceSettings.insert({ attr.name(), attr.value() }); - } auto selectors = settings.Selectors().StringValue(); if (!selectors.empty()) { - source.SetSelectors(selectors); + auto labelValues = NSo::ExtractSelectorValues(selectors); + if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { + labelValues.insert({ "service", settings.Project().StringValue() }); + labelValues.insert({ "cluster", source.GetCluster() }); + } else { + labelValues.insert({ "project", source.GetProject() }); + } + + source.MutableSelectors()->insert(labelValues.begin(), labelValues.end()); } auto program = settings.Program().StringValue(); @@ -345,24 +333,27 @@ public: source.AddRequiredLabelNames(labelAsString); } - auto& solomonSettings = State_->Configuration; + auto defaultReplica = (source.GetClusterType() == NSo::NProto::CT_SOLOMON ? "sas" : "cloud-prod-a"); - auto metricsQueuePageSize = solomonSettings->MetricsQueuePageSize.Get().OrElse(5000); + auto& solomonConfig = State_->Configuration; + auto& sourceSettings = *source.MutableSettings(); + + auto metricsQueuePageSize = solomonConfig->MetricsQueuePageSize.Get().OrElse(5000); sourceSettings.insert({"metricsQueuePageSize", ToString(metricsQueuePageSize)}); - auto metricsQueuePrefetchSize = solomonSettings->MetricsQueuePrefetchSize.Get().OrElse(10000); + auto metricsQueuePrefetchSize = solomonConfig->MetricsQueuePrefetchSize.Get().OrElse(10000); sourceSettings.insert({"metricsQueuePrefetchSize", ToString(metricsQueuePrefetchSize)}); - auto metricsQueueBatchCountLimit = solomonSettings->MetricsQueueBatchCountLimit.Get().OrElse(250); + auto metricsQueueBatchCountLimit = solomonConfig->MetricsQueueBatchCountLimit.Get().OrElse(250); sourceSettings.insert({"metricsQueueBatchCountLimit", ToString(metricsQueueBatchCountLimit)}); - auto solomonClientDefaultReplica = solomonSettings->SolomonClientDefaultReplica.Get().OrElse("sas"); + auto solomonClientDefaultReplica = solomonConfig->SolomonClientDefaultReplica.Get().OrElse(defaultReplica); sourceSettings.insert({"solomonClientDefaultReplica", ToString(solomonClientDefaultReplica)}); - auto computeActorBatchSize = solomonSettings->ComputeActorBatchSize.Get().OrElse(1000); + auto computeActorBatchSize = solomonConfig->ComputeActorBatchSize.Get().OrElse(1000); sourceSettings.insert({"computeActorBatchSize", ToString(computeActorBatchSize)}); - if (!selectors.empty()) { + if (!source.HasProgram()) { auto providerFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, State_->Configuration->Tokens.at(cluster)); auto credentialsProvider = providerFactory->CreateProvider(); @@ -415,7 +406,7 @@ public: shardDesc.SetCluster(shard.Cluster().StringValue()); shardDesc.SetService(shard.Service().StringValue()); - shardDesc.SetClusterType(MapClusterType(clusterDesc->GetClusterType())); + shardDesc.SetClusterType(NSo::MapClusterType(clusterDesc->GetClusterType())); shardDesc.SetUseSsl(clusterDesc->GetUseSsl()); const TTypeAnnotationNode* itemType = shard.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp index 0a6a3c2ac2b..1c526525dbc 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_io_discovery.cpp @@ -55,7 +55,7 @@ const TStructExprType* BuildScheme(TPositionHandle pos, const TVector<TCoAtom>& if (systemColumn == SOLOMON_SCHEME_TS) { type = ctx.MakeType<TDataExprType>(EDataSlot::Datetime); } else if (systemColumn == SOLOMON_SCHEME_VALUE) { - type = ctx.MakeType<TDataExprType>(EDataSlot::Double); + type = ctx.MakeType<TOptionalExprType>(ctx.MakeType<TDataExprType>(EDataSlot::Double)); } else if (systemColumn == SOLOMON_SCHEME_LABELS) { type = ctx.MakeType<NYql::TDictExprType>(stringType, stringType); } else if (systemColumn = SOLOMON_SCHEME_TYPE) { diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp index 67982a34eb9..eaa780a2e31 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp @@ -1,6 +1,7 @@ #include "yql_solomon_provider_impl.h" #include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h> +#include <ydb/library/yql/providers/solomon/common/util.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> #include <ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h> #include <yql/essentials/core/yql_expr_optimize.h> @@ -22,17 +23,6 @@ TMaybe<TString> ExtractSetting(const TExprNode& settings, const TString& setting return {}; } -NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) { - switch (clusterType) { - case TSolomonClusterConfig::SCT_SOLOMON: - return NSo::NProto::ESolomonClusterType::CT_SOLOMON; - case TSolomonClusterConfig::SCT_MONITORING: - return NSo::NProto::ESolomonClusterType::CT_MONITORING; - default: - YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterType)); - } -} - } // namespace class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase { @@ -69,20 +59,25 @@ public: auto& settings = soReadObject.Object().Settings().Ref(); if (auto maybeSelectors = ExtractSetting(settings, "selectors")) { - NSo::NProto::TDqSolomonSource source; - source.SetEndpoint(clusterDesc->GetCluster()); - source.SetProject(soReadObject.Object().Project().StringValue()); - source.SetClusterType(MapClusterType(clusterDesc->GetClusterType())); - source.SetUseSsl(clusterDesc->GetUseSsl()); + NSo::NProto::TDqSolomonSource source = NSo::FillSolomonSource(clusterDesc, soReadObject.Object().Project().StringValue()); + + auto selectors = NSo::ExtractSelectorValues(*maybeSelectors); + if (source.GetClusterType() == NSo::NProto::CT_MONITORING) { + selectors["cluster"] = source.GetCluster(); + selectors["service"] = soReadObject.Object().Project().StringValue(); + } else { + selectors["project"] = source.GetProject(); + } - auto defaultReplica = State_->Configuration->SolomonClientDefaultReplica.Get().OrElse("sas"); - source.MutableSettings()->insert({ "solomonClientDefaultReplica", ToString(defaultReplica) }); + auto defaultReplica = (source.GetClusterType() == NSo::NProto::CT_SOLOMON ? "sas" : "cloud-prod-a"); + auto solomonClientDefaultReplica = State_->Configuration->SolomonClientDefaultReplica.Get().OrElse(defaultReplica); + source.MutableSettings()->insert({ "solomonClientDefaultReplica", ToString(solomonClientDefaultReplica) }); auto providerFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, State_->Configuration->Tokens.at(clusterName)); auto credentialsProvider = providerFactory->CreateProvider(); SolomonClient_ = NSo::ISolomonAccessorClient::Make(std::move(source), credentialsProvider); - auto future = SolomonClient_->GetLabelNames(*maybeSelectors); + auto future = SolomonClient_->GetLabelNames(selectors); LabelNamesRequests_[soReadObject.Raw()] = future; futures.push_back(future); diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp index 3dc8ad9a36c..08983b14ec4 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp @@ -13,12 +13,10 @@ #include <ydb/library/yql/providers/solomon/solomon_accessor/grpc/data_service.grpc.pb.h> #include <util/string/join.h> -#include <util/string/strip.h> -#include <util/string/split.h> namespace NYql::NSo { -using namespace yandex::monitoring::api::v3; +using namespace yandex::cloud::priv::monitoring::v3; namespace { @@ -72,17 +70,7 @@ TString MetricTypeToString(MetricType type) { } } -std::vector<TString> ParseKnownLabelNames(const TString& selectors) { - auto selectorValues = StringSplitter(selectors.substr(1, selectors.size() - 2)).Split(',').SkipEmpty().ToList<TString>(); - std::vector<TString> result; - result.reserve(selectorValues.size()); - for (const auto& value : selectorValues) { - result.push_back(StripString(value.substr(0, value.find('=')))); - } - return result; -} - -TGetLabelsResponse ProcessGetLabelsResponse(NYql::IHTTPGateway::TResult&& response, std::vector<TString>&& knownLabelNames) { +TGetLabelsResponse ProcessGetLabelsResponse(NYql::IHTTPGateway::TResult&& response, const std::map<TString, TString>& knownSelectors) { TGetLabelsResult result; if (response.CurlResponseCode != CURLE_OK) { @@ -113,7 +101,9 @@ TGetLabelsResponse ProcessGetLabelsResponse(NYql::IHTTPGateway::TResult&& respon result.Labels.push_back(name.GetString()); } } - result.Labels.insert(result.Labels.end(), knownLabelNames.begin(), knownLabelNames.end()); + for (const auto& [key, value] : knownSelectors) { + result.Labels.push_back(key); + } return TGetLabelsResponse(std::move(result)); } @@ -141,11 +131,14 @@ TListMetricsResponse ProcessListMetricsResponse(NYql::IHTTPGateway::TResult&& re } const auto pagesInfo = json["page"]; - if (!pagesInfo.IsMap() || !pagesInfo.Has("pagesCount") || !pagesInfo["pagesCount"].IsInteger()) { + if (!pagesInfo.IsMap() || + !pagesInfo.Has("pagesCount") || !pagesInfo["pagesCount"].IsInteger() || + !pagesInfo.Has("totalCount") || !pagesInfo["totalCount"].IsInteger()) { return TListMetricsResponse("Invalid paging info from monitoring api"); } result.PagesCount = pagesInfo["pagesCount"].GetInteger(); + result.TotalCount = pagesInfo["totalCount"].GetInteger(); for (const auto& metricObj : json["result"].GetArray()) { if (!metricObj.IsMap() || !metricObj.Has("labels") || !metricObj["labels"].IsMap() || !metricObj.Has("type") || !metricObj["type"].IsString()) { @@ -163,7 +156,7 @@ TListMetricsResponse ProcessListMetricsResponse(NYql::IHTTPGateway::TResult&& re return TListMetricsResponse(std::move(result)); } -TGetPointsCountResponse ProcessGetPointsCountResponse(NYql::IHTTPGateway::TResult&& response, size_t expectedSize) { +TGetPointsCountResponse ProcessGetPointsCountResponse(NYql::IHTTPGateway::TResult&& response) { TGetPointsCountResult result; if (response.CurlResponseCode != CURLE_OK) { @@ -181,21 +174,11 @@ TGetPointsCountResponse ProcessGetPointsCountResponse(NYql::IHTTPGateway::TResul return TGetPointsCountResponse(TStringBuilder{} << "Failed to parse points count response from monitoring api: " << e.what()); } - if (!json.Has("vector") || !json["vector"].IsArray()) { + if (!json.IsMap() || !json.Has("scalar") || !json["scalar"].IsInteger()) { return TGetPointsCountResponse("Invalid points count result from monitoring api"); } - const auto counts = json["vector"].GetArray(); - if (counts.size() != expectedSize) { - return TGetPointsCountResponse("Invalid points count response size from monitoring api"); - } - - for (size_t i = 0; i < counts.size(); ++i) { - if (!counts[i].IsMap() || !counts[i].Has("scalar") || !counts[i].GetMap().at("scalar").IsInteger()) { - return TGetPointsCountResponse("Invalid points count response format from monitoring api"); - } - result.PointsCount.push_back(counts[i].GetMap().at("scalar").GetInteger()); - } + result.PointsCount = json["scalar"].GetInteger(); return TGetPointsCountResponse(std::move(result)); } @@ -238,11 +221,9 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable public: TSolomonAccessorClient( const TString& defaultReplica, - const ui64 defaultGrpcPort, NYql::NSo::NProto::TDqSolomonSource&& settings, std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider) : DefaultReplica(defaultReplica) - , DefaultGrpcPort(defaultGrpcPort) , Settings(std::move(settings)) , CredentialsProvider(credentialsProvider) { @@ -261,22 +242,26 @@ public: } public: - NThreading::TFuture<TGetLabelsResponse> GetLabelNames(const TString& selectors) const override final { + NThreading::TFuture<TGetLabelsResponse> GetLabelNames(const std::map<TString, TString>& selectors) const override final { + auto requestUrl = BuildGetLabelsUrl(selectors); + auto resultPromise = NThreading::NewPromise<TGetLabelsResponse>(); auto cb = [resultPromise, selectors](NYql::IHTTPGateway::TResult&& result) mutable { - resultPromise.SetValue(ProcessGetLabelsResponse(std::move(result), ParseKnownLabelNames(selectors))); + resultPromise.SetValue(ProcessGetLabelsResponse(std::move(result), selectors)); }; DoHttpRequest( std::move(cb), - BuildGetLabelsUrl(selectors) + std::move(requestUrl) ); return resultPromise.GetFuture(); } - NThreading::TFuture<TListMetricsResponse> ListMetrics(const TString& selectors, int pageSize, int page) const override final { + NThreading::TFuture<TListMetricsResponse> ListMetrics(const std::map<TString, TString>& selectors, int pageSize, int page) const override final { + auto requestUrl = BuildListMetricsUrl(selectors, pageSize, page); + auto resultPromise = NThreading::NewPromise<TListMetricsResponse>(); auto cb = [resultPromise](NYql::IHTTPGateway::TResult&& result) mutable { @@ -285,20 +270,23 @@ public: DoHttpRequest( std::move(cb), - BuildListMetricsUrl(selectors, pageSize, page) + std::move(requestUrl) ); return resultPromise.GetFuture(); } - NThreading::TFuture<TGetPointsCountResponse> GetPointsCount(const std::vector<TMetric>& metrics) const override final { + NThreading::TFuture<TGetPointsCountResponse> GetPointsCount(const std::map<TString, TString>& selectors) const override final { + auto fullSelectors = AddRequiredLabels(selectors); + TString program = TStringBuilder() << "count(" << BuildSelectorsProgram(fullSelectors) << ")"; + auto requestUrl = BuildGetPointsCountUrl(); - auto requestBody = BuildGetPointsCountBody(metrics); + auto requestBody = BuildGetPointsCountBody(program); auto resultPromise = NThreading::NewPromise<TGetPointsCountResponse>(); - auto cb = [resultPromise, expectedSize = metrics.size()](NYql::IHTTPGateway::TResult&& response) mutable { - resultPromise.SetValue(ProcessGetPointsCountResponse(std::move(response), expectedSize)); + auto cb = [resultPromise](NYql::IHTTPGateway::TResult&& response) mutable { + resultPromise.SetValue(ProcessGetPointsCountResponse(std::move(response))); }; DoHttpRequest( @@ -310,12 +298,22 @@ public: return resultPromise.GetFuture(); } - NThreading::TFuture<TGetDataResponse> GetData(TMetric metric, TInstant from, TInstant to) const override final { - return GetData(BuildSelectorsProgram(metric), from, to); + NThreading::TFuture<TGetDataResponse> GetData(const std::map<TString, TString>& selectors, TInstant from, TInstant to) const override final { + auto fullSelectors = AddRequiredLabels(selectors); + + if (Settings.GetClusterType() == NProto::CT_MONITORING) { + fullSelectors["folderId"] = fullSelectors["cluster"]; + fullSelectors.erase("cluster"); + fullSelectors.erase("project"); + } + + TString program = BuildSelectorsProgram(fullSelectors, true); + + return GetData(program, from, to); } - NThreading::TFuture<TGetDataResponse> GetData(TString selectors, TInstant from, TInstant to) const override final { - const auto request = BuildGetDataRequest(selectors, from, to); + NThreading::TFuture<TGetDataResponse> GetData(TString program, TInstant from, TInstant to) const override final { + const auto request = BuildGetDataRequest(program, from, to); NYdbGrpc::TCallMeta callMeta; if (auto authInfo = GetAuthInfo()) { @@ -366,11 +364,11 @@ private: } TString GetHttpSolomonEndpoint() const { - return (Settings.GetUseSsl() ? "https://" : "http://") + Settings.GetEndpoint(); + return TStringBuilder() << (Settings.GetUseSsl() ? "https://" : "http://") << Settings.GetHttpEndpoint(); } TString GetGrpcSolomonEndpoint() const { - return TStringBuilder() << Settings.GetEndpoint() << ":" << DefaultGrpcPort; + return TStringBuilder() << Settings.GetGrpcEndpoint(); } template <typename TCallback> @@ -417,7 +415,7 @@ private: } } - TString BuildGetLabelsUrl(const TString& selectors) const { + TString BuildGetLabelsUrl(const std::map<TString, TString>& selectors) const { TUrlBuilder builder(GetHttpSolomonEndpoint()); builder.AddPathComponent("api"); @@ -427,13 +425,13 @@ private: builder.AddPathComponent("sensors"); builder.AddPathComponent("names"); - builder.AddUrlParam("selectors", selectors); + builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors)); builder.AddUrlParam("forceCluster", DefaultReplica); return builder.Build(); } - TString BuildListMetricsUrl(const TString& selectors, int pageSize, int page) const { + TString BuildListMetricsUrl(const std::map<TString, TString>& selectors, int pageSize, int page) const { TUrlBuilder builder(GetHttpSolomonEndpoint()); builder.AddPathComponent("api"); @@ -442,12 +440,10 @@ private: builder.AddPathComponent(Settings.GetProject()); builder.AddPathComponent("sensors"); - builder.AddUrlParam("selectors", selectors); + builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors)); builder.AddUrlParam("forceCluster", DefaultReplica); builder.AddUrlParam("pageSize", std::to_string(pageSize)); builder.AddUrlParam("page", std::to_string(page)); - builder.AddUrlParam("from", TInstant::Seconds(Settings.GetFrom()).ToIsoStringLocalUpToSeconds()); - builder.AddUrlParam("to", TInstant::Seconds(Settings.GetTo()).ToIsoStringLocalUpToSeconds()); return builder.Build(); } @@ -465,46 +461,37 @@ private: return builder.Build(); } - TString BuildGetPointsCountBody(const std::vector<TMetric>& metrics) const { - std::vector<TString> selectors; - selectors.reserve(metrics.size()); - std::transform( - metrics.begin(), - metrics.end(), - std::back_inserter(selectors), - [this](const TMetric& metric) -> TString { - return TStringBuilder() << "count(" << BuildSelectorsProgram(metric) << ")"; - } - ); - - TString program = TStringBuilder() << "[" << JoinSeq(",", selectors) << "]"; - + TString BuildGetPointsCountBody(const TString& program) const { const auto& ds = Settings.GetDownsampling(); NJsonWriter::TBuf w; w.BeginObject() .UnsafeWriteKey("from").WriteString(TInstant::Seconds(Settings.GetFrom()).ToString()) .UnsafeWriteKey("to").WriteString(TInstant::Seconds(Settings.GetTo()).ToString()) .UnsafeWriteKey("program").WriteString(program) + .UnsafeWriteKey("forceCluster").WriteString(DefaultReplica) .UnsafeWriteKey("downsampling") .BeginObject() .UnsafeWriteKey("disabled").WriteBool(ds.GetDisabled()); if (!ds.GetDisabled()) { - w.UnsafeWriteKey("aggregation").WriteString(ds.GetAggregation()) + w + .UnsafeWriteKey("aggregation").WriteString(ds.GetAggregation()) .UnsafeWriteKey("fill").WriteString(ds.GetFill()) .UnsafeWriteKey("gridMillis").WriteLongLong(ds.GetGridMs()); } - w.EndObject() - .UnsafeWriteKey("forceCluster").WriteString(DefaultReplica) - .EndObject(); + w.EndObject().EndObject(); return w.Str(); } - ReadRequest BuildGetDataRequest(const TString& selectors, TInstant from, TInstant to) const { + ReadRequest BuildGetDataRequest(const TString& program, TInstant from, TInstant to) const { ReadRequest request; - request.mutable_container()->set_project_id(Settings.GetProject()); + if (Settings.GetClusterType() == NProto::CT_SOLOMON) { + request.mutable_container()->set_project_id(Settings.GetProject()); + } else { + request.mutable_container()->set_folder_id(Settings.GetCluster()); + } *request.mutable_from_time() = NProtoInterop::CastToProto(from); *request.mutable_to_time() = NProtoInterop::CastToProto(to); *request.mutable_force_replica() = DefaultReplica; @@ -519,37 +506,47 @@ private: } auto query = request.mutable_queries()->Add(); - *query->mutable_value() = selectors; + *query->mutable_value() = program; *query->mutable_name() = "query"; query->set_hidden(false); return request; } - TString BuildSelectorsProgram(const TMetric& metric) const { + std::map<TString, TString> AddRequiredLabels(const std::map<TString, TString>& labels) const { + std::map<TString, TString> fullSelectors; + for (const auto& labelName : Settings.GetRequiredLabelNames()) { + if (auto it = labels.find(labelName); it != labels.end()) { + fullSelectors[labelName] = it->second; + } else { + fullSelectors[labelName] = "-"; + } + } + return fullSelectors; + } + + TString BuildSelectorsProgram(const std::map<TString, TString>& labels, bool useNewFormat = false) const { std::vector<TString> mappedValues; - mappedValues.reserve(Settings.GetRequiredLabelNames().size()); - std::transform( - Settings.GetRequiredLabelNames().begin(), - Settings.GetRequiredLabelNames().end(), - std::back_inserter(mappedValues), - [&labels = metric.Labels](const TString& labelName) -> TString { - if (labels.find(labelName) == labels.end()) { - return TStringBuilder() << labelName << "=\"-\""; - } - return TStringBuilder() << labelName << "=\"" << labels.at(labelName) << "\""; + for (const auto& [key, value] : labels) { + if (useNewFormat && key == "name"sv) { + continue; } - ); + mappedValues.push_back(TStringBuilder() << key << "=\"" << value << "\""); + } - return TStringBuilder() << "{" << JoinSeq(",", mappedValues) << "}"; + TStringBuilder result; + if (auto it = labels.find("name"); useNewFormat && it != labels.end()) { + result << "\"" << it->second << "\""; + } + + return result << "{" << JoinSeq(",", mappedValues) << "}"; } private: const TString DefaultReplica; - const ui64 DefaultGrpcPort; const ui64 ListSizeLimit = 1ull << 20; - const ui64 HttpMaxInflight = 200; - const ui64 GrpcMaxInflight = 2000; + const ui64 HttpMaxInflight = 1000; + const ui64 GrpcMaxInflight = 1000; const NYql::NSo::NProto::TDqSolomonSource Settings; const std::shared_ptr<NYdb::ICredentialsProvider> CredentialsProvider; @@ -568,17 +565,12 @@ ISolomonAccessorClient::Make( std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider) { const auto& settings = source.settings(); - TString defaultReplica = "sas"; + TString defaultReplica; if (auto it = settings.find("solomonClientDefaultReplica"); it != settings.end()) { defaultReplica = it->second; } - ui64 defaultGrpcPort = 443; - if (auto it = settings.find("grpcPort"); it != settings.end()) { - defaultGrpcPort = FromString<ui64>(it->second); - } - - return std::make_shared<TSolomonAccessorClient>(defaultReplica, defaultGrpcPort, std::move(source), credentialsProvider); + return std::make_shared<TSolomonAccessorClient>(defaultReplica, std::move(source), credentialsProvider); } } // namespace NYql::NSo diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h index 87e1728e51c..f90d2dda4e1 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.h @@ -19,10 +19,10 @@ public: std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider); public: - virtual NThreading::TFuture<TGetLabelsResponse> GetLabelNames(const TString& selectors) const = 0; - virtual NThreading::TFuture<TListMetricsResponse> ListMetrics(const TString& selectors, int pageSize, int page) const = 0; - virtual NThreading::TFuture<TGetPointsCountResponse> GetPointsCount(const std::vector<TMetric>& metrics) const = 0; - virtual NThreading::TFuture<TGetDataResponse> GetData(TMetric metric, TInstant from, TInstant to) const = 0; + virtual NThreading::TFuture<TGetLabelsResponse> GetLabelNames(const std::map<TString, TString>& selectors) const = 0; + virtual NThreading::TFuture<TListMetricsResponse> ListMetrics(const std::map<TString, TString>& selectors, int pageSize, int page) const = 0; + virtual NThreading::TFuture<TGetPointsCountResponse> GetPointsCount(const std::map<TString, TString>& selectors) const = 0; + virtual NThreading::TFuture<TGetDataResponse> GetData(const std::map<TString, TString>& selectors, TInstant from, TInstant to) const = 0; virtual NThreading::TFuture<TGetDataResponse> GetData(TString selectors, TInstant from, TInstant to) const = 0; }; diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.cpp b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.cpp index 5356e508288..0d1450a1627 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.cpp +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.cpp @@ -1,5 +1,9 @@ #include "solomon_client_utils.h" +#include <util/string/join.h> +#include <util/string/strip.h> +#include <util/string/split.h> + #include <yql/essentials/utils/yql_panic.h> namespace NYql::NSo { diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.h b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.h index 6682d4e5989..5115959116c 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.h +++ b/ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_client_utils.h @@ -48,11 +48,12 @@ using TGetLabelsResponse = TSolomonClientResponse<TGetLabelsResult>; struct TListMetricsResult { std::vector<TMetric> Metrics; ui64 PagesCount; + ui64 TotalCount; }; using TListMetricsResponse = TSolomonClientResponse<TListMetricsResult>; struct TGetPointsCountResult { - std::vector<ui64> PointsCount; + ui64 PointsCount; }; using TGetPointsCountResponse = TSolomonClientResponse<TGetPointsCountResult>; diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/grpc/data_service.proto b/ydb/library/yql/providers/solomon/solomon_accessor/grpc/data_service.proto index 947b1675ece..c866f0ef4f2 100644 --- a/ydb/library/yql/providers/solomon/solomon_accessor/grpc/data_service.proto +++ b/ydb/library/yql/providers/solomon/solomon_accessor/grpc/data_service.proto @@ -1,11 +1,12 @@ syntax = "proto3"; -package yandex.monitoring.api.v3; +package yandex.cloud.priv.monitoring.v3; import "google/protobuf/timestamp.proto"; +import "google/protobuf/descriptor.proto"; import "google/rpc/status.proto"; -option go_package = "github.com/ydb-platform/ydb/ydb/library/yql/providers/solomon/solomon_accessor/grpc;solomon_accessor_pb"; +option go_package = "github.com/ydb-platform/ydb/ydb/library/yql/providers/solomon/solomon_accessor/grpc;data_service"; // A set of methods for reading metrics data. service DataService { @@ -14,6 +15,82 @@ service DataService { } } +message ReadRequest { + // Default container (project/folder) for this request. + Container container = 1; + + // Time from which data should be collected. + google.protobuf.Timestamp from_time = 3; + + // Time until which data should be collected. + google.protobuf.Timestamp to_time = 4; + + // Queries which are data gathered based on. + repeated Query queries = 5; + + // Downsampling parameters. + Downsampling downsampling = 6; + + // Force cluster. + string force_replica = 7; +} + +message ReadResponse { + // Data response for each not hidden query. + repeated ResponsePerQuery response_per_query = 1; + + // Error raised during request processing. + repeated google.rpc.Status errors = 2; +} + +message ResponsePerQuery { + message TimeseriesVector { + // Timeseries values. + repeated TimeseriesValue values = 1; + } + + // Name of the query. + string query_name = 1; + + // Result values for the query. + oneof result_values { + // Timeseries vector. + TimeseriesVector timeseries_vector = 2; + } +} + +message TimeseriesValue { + // Alias. + string alias = 1; + + // Name of the metric. + string name = 2; + + // List of metric labels as `key:value` pairs. + map<string, string> labels = 3; + + // Type of the metric. + MetricType type = 4; + + // List of timestamps. + Int64Vector timestamp_values = 5; + + oneof values { + // List of double values. + DoubleVector double_values = 6; + } +} + +message DoubleVector { + // List of double values. + repeated double values = 1; +} + +message Int64Vector { + // List of int64 values. + repeated int64 values = 1; +} + message Container { // Default project/folder for this request. oneof container { @@ -27,6 +104,20 @@ message Container { } } +// Query used in multi-source requests for metrics retrieving. +message Query { + // Name of the query. + string name = 1; + + // Text of the program that will be executed. + string value = 2; + + // Flag for hiding result timeseries from response. + // Note: logic will still be executed, + // so result of the query processing can be used in following calculations + bool hidden = 3; +} + // List of available aggregate functions for downsampling. message Downsampling { // List of available aggregate functions for downsampling. @@ -85,64 +176,6 @@ message Downsampling { GapFilling gap_filling = 5; } -// Query used in multi-source requests for metrics retrieving. -message Query { - // Name of the query. - string name = 1; - - // Text of the program that will be executed. - string value = 2; - - // Flag for hiding result timeseries from response. - // Note: logic will still be executed, - // so result of the query processing can be used in following calculations - bool hidden = 3; -} - -message ReadRequest { - // Default container (project/folder) for this request. - Container container = 1; - - // Time from which data should be collected. - google.protobuf.Timestamp from_time = 3; - - // Time until which data should be collected. - google.protobuf.Timestamp to_time = 4; - - // Queries which are data gathered based on. - repeated Query queries = 5; - - // Downsampling parameters. - Downsampling downsampling = 6; - - // Force cluster. - string force_replica = 7; -} - -message ReadResponse { - // Data response for each not hidden query. - repeated ResponsePerQuery response_per_query = 1; - - // Error raised during request processing. - repeated google.rpc.Status errors = 2; -} - -message ResponsePerQuery { - message TimeseriesVector { - // Timeseries values. - repeated TimeseriesValue values = 1; - } - - // Name of the query. - string query_name = 1; - - // Result values for the query. - oneof result_values { - // Timeseries vector. - TimeseriesVector timeseries_vector = 2; - } -} - enum MetricType { METRIC_TYPE_UNSPECIFIED = 0; @@ -158,35 +191,3 @@ enum MetricType { // Rate. RATE = 4; } - -message TimeseriesValue { - // Alias. - string alias = 1; - - // Name of the metric. - string name = 2; - - // List of metric labels as `key:value` pairs. - map<string, string> labels = 3; - - // Type of the metric. - MetricType type = 4; - - // List of timestamps. - Int64Vector timestamp_values = 5; - - oneof values { - // List of double values. - DoubleVector double_values = 6; - } -} - -message DoubleVector { - // List of double values. - repeated double values = 1; -} - -message Int64Vector { - // List of int64 values. - repeated int64 values = 1; -} diff --git a/ydb/library/yql/providers/solomon/solomon_accessor/grpc/solomon_accessor_pb.proto b/ydb/library/yql/providers/solomon/solomon_accessor/grpc/solomon_accessor_pb.proto deleted file mode 100644 index e8b076fc697..00000000000 --- a/ydb/library/yql/providers/solomon/solomon_accessor/grpc/solomon_accessor_pb.proto +++ /dev/null @@ -1,194 +0,0 @@ -syntax = "proto3"; - -package yandex.monitoring.api.v3; - -import "google/protobuf/duration.proto"; -import "google/protobuf/timestamp.proto"; -import "google/rpc/status.proto"; - -option go_package = "github.com/ydb-platform/ydb/ydb/library/yql/providers/solomon/solomon_accessor/grpc;solomon_accessor_pb"; - -// A set of methods for reading metrics data. -service DataService { - // Retrieves metric data based on multi-query input. - rpc Read(ReadRequest) returns (ReadResponse) { - } - -} - -message Container { - // Default project/folder for this request. - oneof container { - // ID of the project that will be used as default. - // (unless it's specified in query) - string project_id = 1; - - // Cloud folder that will be used as default. - // (unless it's specified in query) - string folder_id = 2; - } -} - -// List of available aggregate functions for downsampling. -message Downsampling { - // List of available aggregate functions for downsampling. - enum GridAggregation { - GRID_AGGREGATION_UNSPECIFIED = 0; - - // Max value. - GRID_AGGREGATION_MAX = 1; - - // Min value. - GRID_AGGREGATION_MIN = 2; - - // Sum of values. - GRID_AGGREGATION_SUM = 3; - - // Average value. - GRID_AGGREGATION_AVG = 4; - - // Last value. - GRID_AGGREGATION_LAST = 5; - - // Total count of points. - GRID_AGGREGATION_COUNT = 6; - } - - // List of available gap filling policy for downsampling. - enum GapFilling { - GAP_FILLING_UNSPECIFIED = 0; - - // Returns `null` as a metric value and `timestamp` as a time series value. - GAP_FILLING_NULL = 1; - - // Returns no value and no timestamp. - GAP_FILLING_NONE = 2; - - // Returns the value from the previous time interval. - GAP_FILLING_PREVIOUS = 3; - } - - oneof mode { - // Maximum number of points to be returned. - int64 max_points = 1; - - // Time interval (grid) for downsampling in milliseconds. - // Points in the specified range are aggregated into one time point. - int64 grid_interval = 2; - - // Disable downsampling. - bool disabled = 3; - } - - // Function that is used for downsampling. - GridAggregation grid_aggregation = 4; - - // Parameters for filling gaps in data. - GapFilling gap_filling = 5; -} - -// Query used in multi-source requests for metrics retrieving. -message Query { - // Name of the query. - string name = 1; - - // Text of the program that will be executed. - string value = 2; - - // Flag for hiding result timeseries from response. - // Note: logic will still be executed, - // so result of the query processing can be used in following calculations - bool hidden = 3; -} - -message ReadRequest { - // Default container (project/folder) for this request. - Container container = 1; - - // Time from which data should be collected. - google.protobuf.Timestamp from_time = 3; - - // Time until which data should be collected. - google.protobuf.Timestamp to_time = 4; - - // Queries which are data gathered based on. - repeated Query queries = 5; - - // Downsampling parameters. - Downsampling downsampling = 6; - - // Force cluster. - string force_replica = 7; -} - -message ReadResponse { - // Data response for each not hidden query. - repeated ResponsePerQuery response_per_query = 1; - - // Error raised during request processing. - repeated google.rpc.Status errors = 2; -} - -message ResponsePerQuery { - message TimeseriesVector { - // Timeseries values. - repeated TimeseriesValue values = 1; - } - - // Name of the query. - string query_name = 1; - - // Result values for the query. - oneof result_values { - // Timeseries vector. - TimeseriesVector timeseries_vector = 2; - } -} - -enum MetricType { - METRIC_TYPE_UNSPECIFIED = 0; - - // Gauge with fractional values. - DGAUGE = 1; - - // Gauge with integer values. - IGAUGE = 2; - - // Counter. - COUNTER = 3; - - // Rate. - RATE = 4; -} - -message TimeseriesValue { - // Alias. - string alias = 1; - - // Name of the metric. - string name = 2; - - // List of metric labels as `key:value` pairs. - map<string, string> labels = 3; - - // Type of the metric. - MetricType type = 4; - - // List of timestamps. - Int64Vector timestamp_values = 5; - - oneof values { - // List of double values. - DoubleVector double_values = 6; - } -} - -message DoubleVector { - // List of double values. - repeated double values = 1; -} - -message Int64Vector { - // List of int64 values. - repeated int64 values = 1; -} diff --git a/ydb/library/yql/providers/solomon/ya.make b/ydb/library/yql/providers/solomon/ya.make index 18d6ac7cc37..d75ebdd9142 100644 --- a/ydb/library/yql/providers/solomon/ya.make +++ b/ydb/library/yql/providers/solomon/ya.make @@ -1,5 +1,6 @@ RECURSE( actors + common events expr_nodes gateway diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql index 96756a09c5b..cd3deb8a589 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/opt.yql @@ -2,7 +2,7 @@ (let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) (let $2 (DataSink 'result)) (let $3 (DataType 'String)) -(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (DataType 'Double)))) +(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) (let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt index 9f274245d94..59ad016e435 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["DataType";"Double"]]]]];"Data"=[[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"100"];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"200"];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"300"]]}]}]
\ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}]}]
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql index 4a755e7e36f..cb4a7dc534a 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/opt.yql @@ -2,7 +2,7 @@ (let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) (let $2 (DataSink 'result)) (let $3 (DataType 'String)) -(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (DataType 'Double)))) +(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) (let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt index 86a8633bdf5..9defd3e8b2d 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["DataType";"Double"]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"100"];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"200"];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"300"]]}]}]
\ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}]}]
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql index 1a969a8de67..48b044dde14 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/opt.yql @@ -2,7 +2,7 @@ (let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) (let $2 (DataSink 'result)) (let $3 (DataType 'String)) -(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (DataType 'Double)))) +(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) (let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"15") '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt index 47eaf7656dc..a616aa78844 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["DataType";"Double"]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"100"];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"200"];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"300"]]}]}]
\ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}]}]
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql index 6c61ea29c43..d4e5bc5b3ee 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/opt.yql @@ -2,7 +2,7 @@ (let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) (let $2 (DataSink 'result)) (let $3 (DataType 'String)) -(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (DataType 'Double)))) +(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) (let $6 (SoSourceSettings world '"hist" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"histogram_percentile(95, {})" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($10) $10) '('('"_logical_id" '0)))) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt index d2613b2aa63..568abc9e6d8 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["DataType";"Double"]]]]];"Data"=[[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";"100"];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";"200"];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";"300"]]}]}]
\ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"4"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 15000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 28"];["project";"hist"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}]}]
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql index ba0fe486fab..a8171a4d250 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/opt.yql @@ -5,7 +5,7 @@ (let $4 (DataType 'String)) (let $5 '"downsampling.fill") (let $6 '"downsampling.gridMillis") -(let $7 (StructType '($3 $4) '($5 $4) '($6 $4) '('"project" $4) '('"ts" (DataType 'Datetime)) '('type $4) '('"value" (DataType 'Double)))) +(let $7 (StructType '($3 $4) '($5 $4) '($6 $4) '('"project" $4) '('"ts" (DataType 'Datetime)) '('type $4) '('"value" (OptionalType (DataType 'Double))))) (let $8 '('"value" '"ts" 'type)) (let $9 '($3 $5 '"project" $6)) (let $10 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $7 $8 $9 '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"AVG" '"PREVIOUS" (Uint32 '"15") '())) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt index 9fe98752882..a3886d17a7f 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["downsampling.disabled";["DataType";"String"]];["downsampling.fill";["DataType";"String"]];["downsampling.gridMillis";["DataType";"String"]];["project";["DataType";"String"]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["DataType";"Double"]]]]];"Data"=[["bool False";"3";"int 15000";"my_project";"0";"RATE";"100"];["bool False";"3";"int 15000";"my_project";"0";"RATE";"200"];["bool False";"3";"int 15000";"my_project";"0";"RATE";"300"]]}]}]
\ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["downsampling.disabled";["DataType";"String"]];["downsampling.fill";["DataType";"String"]];["downsampling.gridMillis";["DataType";"String"]];["project";["DataType";"String"]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[["bool False";"3";"int 15000";"my_project";"0";"RATE";["100"]];["bool False";"3";"int 15000";"my_project";"0";"RATE";["200"]];["bool False";"3";"int 15000";"my_project";"0";"RATE";["300"]]]}]}]
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql index 508f856cab9..5f7da55b66b 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/opt.yql @@ -2,7 +2,7 @@ (let $1 (Configure! world (DataSource '"config") '"DqEngine" '"force")) (let $2 (DataSink 'result)) (let $3 (DataType 'String)) -(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (DataType 'Double)))) +(let $4 (StructType '('"labels" (DictType $3 $3)) '('"ts" (DataType 'Datetime)) '('type $3) '('"value" (OptionalType (DataType 'Double))))) (let $5 '('"labels" '"value" '"ts" 'type)) (let $6 (SoSourceSettings world '"my_project" (SecureParam '"cluster:default_local_solomon") $4 $5 '() '"2023-12-08T14:40:39Z" '"2023-12-08T14:45:39Z" '"" '"{}" (Bool '"false") '"SUM" '"PREVIOUS" (Uint32 '"25") '())) (let $7 (DqStage '((DqSource (DataSource '"solomon" '"local_solomon") $6)) (lambda '($11) $11) '('('"_logical_id" '0)))) diff --git a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/results.txt b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/results.txt index 2ef521b92f7..a8106499556 100644 --- a/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/results.txt +++ b/ydb/library/yql/tests/sql/solomon/canondata/test.test_solomon-Subquery-default.txt_/results.txt @@ -1 +1 @@ -[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["DataType";"Double"]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"100"];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"200"];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";"300"]]}];"Unordered"=%true}]
\ No newline at end of file +[{"Write"=[{"Type"=["ListType";["StructType";[["labels";["DictType";["DataType";"String"];["DataType";"String"]]];["ts";["DataType";"Datetime"]];["type";["DataType";"String"]];["value";["OptionalType";["DataType";"Double"]]]]]];"Data"=[[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["100"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["200"]];[[["downsampling.aggregation";"3"];["downsampling.fill";"3"];["downsampling.gridMillis";"int 25000"];["from";"seconds: 1702046439\n"];["downsampling.disabled";"bool False"];["program";"program length 2"];["project";"my_project"];["to";"seconds: 1702046739\n"]];"0";"RATE";["300"]]]}];"Unordered"=%true}]
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/solomon/test.py b/ydb/library/yql/tests/sql/solomon/test.py index 6678c27201d..e5cc4893522 100644 --- a/ydb/library/yql/tests/sql/solomon/test.py +++ b/ydb/library/yql/tests/sql/solomon/test.py @@ -48,9 +48,6 @@ def compose_gateways_config(solomon_endpoint): solomon_cluster.Name = "local_solomon" solomon_cluster.Cluster = solomon_endpoint solomon_cluster.UseSsl = False - grpcPortSetting = solomon_cluster.Settings.add() - grpcPortSetting.Name = "grpcPort" - grpcPortSetting.Value = os.environ.get("SOLOMON_PORT") return text_format.MessageToString(config_message) diff --git a/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py b/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py index 9ba01f1f6f0..79822568766 100644 --- a/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py +++ b/ydb/library/yql/tools/solomon_emulator_grpc/__main__.py @@ -94,7 +94,7 @@ def start(argv): def _update_environment(port: int): - endpoint = "localhost" + endpoint = f"localhost:{port}" url = "localhost" set_env("SOLOMON_HOST", "localhost") set_env("SOLOMON_PORT", str(port)) diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt index 4332ecc325a..51d34a44bff 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-Basic-default.txt_/results.txt @@ -26,7 +26,11 @@ columns { columns { name: "value" type { - type_id: DOUBLE + optional_type { + item { + type_id: DOUBLE + } + } } } rows { diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt index 238013b2a1b..d74508502b6 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-Downsampling-default.txt_/results.txt @@ -26,7 +26,11 @@ columns { columns { name: "value" type { - type_id: DOUBLE + optional_type { + item { + type_id: DOUBLE + } + } } } rows { diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt index 12e93e15e06..4a16dad9dab 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-DownsamplingValidSettings-default.txt_/results.txt @@ -26,7 +26,11 @@ columns { columns { name: "value" type { - type_id: DOUBLE + optional_type { + item { + type_id: DOUBLE + } + } } } rows { diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt index df47907b0bc..e48e901a8e0 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-HistResponse-default.txt_/results.txt @@ -26,7 +26,11 @@ columns { columns { name: "value" type { - type_id: DOUBLE + optional_type { + item { + type_id: DOUBLE + } + } } } rows { diff --git a/ydb/tests/fq/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt b/ydb/tests/fq/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt index 0df4b7ccac2..2f0927aa4ae 100644 --- a/ydb/tests/fq/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt +++ b/ydb/tests/fq/solomon/canondata/test.test_solomon-LabelColumns-default.txt_/results.txt @@ -37,7 +37,11 @@ columns { columns { name: "value" type { - type_id: DOUBLE + optional_type { + item { + type_id: DOUBLE + } + } } } rows { diff --git a/ydb/tests/fq/solomon/cfg/kqprun_scheme.sql b/ydb/tests/fq/solomon/cfg/kqprun_scheme.sql index f8375d5ca0a..0e3ee16d08c 100644 --- a/ydb/tests/fq/solomon/cfg/kqprun_scheme.sql +++ b/ydb/tests/fq/solomon/cfg/kqprun_scheme.sql @@ -2,6 +2,5 @@ CREATE EXTERNAL DATA SOURCE local_solomon WITH ( SOURCE_TYPE="Solomon", LOCATION="${SOLOMON_ENDPOINT}", AUTH_METHOD="NONE", - USE_SSL="false", - GRPC_PORT="${SOLOMON_PORT}" + USE_SSL="false" ); diff --git a/ydb/tests/fq/solomon/test.py b/ydb/tests/fq/solomon/test.py index 3290135782b..a7fb59b12fd 100644 --- a/ydb/tests/fq/solomon/test.py +++ b/ydb/tests/fq/solomon/test.py @@ -64,7 +64,7 @@ def test(suite, case, cfg, solomon): scheme_file=os.path.join('ydb/tests/fq/solomon/cfg', 'kqprun_scheme.sql')) yqlrun_res = kqprun.yql_exec( yql_program=sql_query, - var_templates=['SOLOMON_ENDPOINT', 'SOLOMON_PORT'], + var_templates=['SOLOMON_ENDPOINT'], verbose=True, check_error=not xfail ) |