diff options
author | d-mokhnatkin <d-mokhnatkin@yandex-team.ru> | 2022-02-10 16:50:43 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:43 +0300 |
commit | d94e393f3d35cf42ec17be6dc35149893a0b49cf (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 4a2fe1d5dc0a49394dec49309de618073ccd14b7 (diff) | |
download | ydb-d94e393f3d35cf42ec17be6dc35149893a0b49cf.tar.gz |
Restoring authorship annotation for <d-mokhnatkin@yandex-team.ru>. Commit 2 of 2.
167 files changed, 6940 insertions, 6940 deletions
diff --git a/library/cpp/actors/http/http_proxy.cpp b/library/cpp/actors/http/http_proxy.cpp index 4b57a8453dc..36c6855d93f 100644 --- a/library/cpp/actors/http/http_proxy.cpp +++ b/library/cpp/actors/http/http_proxy.cpp @@ -1,4 +1,4 @@ -#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/events.h> #include <library/cpp/monlib/metrics/metric_registry.h> #include "http_proxy.h" @@ -43,19 +43,19 @@ protected: HFunc(TEvHttpProxy::TEvHttpConnectionClosed, Handle); HFunc(TEvHttpProxy::TEvResolveHostRequest, Handle); HFunc(TEvHttpProxy::TEvReportSensors, Handle); - HFunc(NActors::TEvents::TEvPoison, Handle); + HFunc(NActors::TEvents::TEvPoison, Handle); } } - void PassAway() override { - Send(Poller, new NActors::TEvents::TEvPoisonPill()); + void PassAway() override { + Send(Poller, new NActors::TEvents::TEvPoisonPill()); for (const NActors::TActorId& connection : Connections) { - Send(connection, new NActors::TEvents::TEvPoisonPill()); + Send(connection, new NActors::TEvents::TEvPoisonPill()); } for (const NActors::TActorId& acceptor : Acceptors) { - Send(acceptor, new NActors::TEvents::TEvPoisonPill()); + Send(acceptor, new NActors::TEvents::TEvPoisonPill()); } - NActors::TActorBootstrapped<THttpProxy>::PassAway(); + NActors::TActorBootstrapped<THttpProxy>::PassAway(); } void Handle(TEvHttpProxy::TEvHttpIncomingRequest::TPtr event, const NActors::TActorContext& ctx) { @@ -200,10 +200,10 @@ protected: NMonitoring::ExplicitHistogram({1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 30000, 60000}))->Record(sensors.Time.MilliSeconds()); } - void Handle(NActors::TEvents::TEvPoison::TPtr, const NActors::TActorContext&) { - PassAway(); - } - + void Handle(NActors::TEvents::TEvPoison::TPtr, const NActors::TActorContext&) { + PassAway(); + } + NActors::TActorId Poller; TVector<TActorId> Acceptors; diff --git a/library/cpp/streams/lz/lz.cpp b/library/cpp/streams/lz/lz.cpp index de090a257c9..b65bb3ed965 100644 --- a/library/cpp/streams/lz/lz.cpp +++ b/library/cpp/streams/lz/lz.cpp @@ -3,7 +3,7 @@ #include <util/system/yassert.h> #include <util/system/byteorder.h> #include <util/memory/addstorage.h> -#include <util/generic/buffer.h> +#include <util/generic/buffer.h> #include <util/generic/utility.h> #include <util/generic/singleton.h> #include <util/generic/yexception.h> @@ -193,8 +193,8 @@ public: , Version_(CheckVer(Load<ui32>())) , BlockSize_(Load<ui16>()) , OutBufSize_(TDecompressor::Hint(BlockSize_)) - , Tmp_(2 * OutBufSize_) - , In_(Tmp_.Data()) + , Tmp_(2 * OutBufSize_) + , In_(Tmp_.Data()) , Out_(In_ + OutBufSize_) { this->InitFromStream(Slave_); @@ -237,9 +237,9 @@ public: TMemoryInput header(tmp, sizeof(tmp)); const ui16 len = GLoad<ui16>(&header); - if (len > Tmp_.Capacity()) { - ythrow TDecompressorError() << "invalid len inside block header"; - } + if (len > Tmp_.Capacity()) { + ythrow TDecompressorError() << "invalid len inside block header"; + } const ui8 compressed = GLoad<ui8>(&header); if (compressed > 1) { @@ -271,7 +271,7 @@ protected: const ui32 Version_; const ui16 BlockSize_; const size_t OutBufSize_; - TBuffer Tmp_; + TBuffer Tmp_; char* In_; char* Out_; }; diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp index 5949f397c1b..6876f070fc0 100644 --- a/library/cpp/streams/lz/lz_ut.cpp +++ b/library/cpp/streams/lz/lz_ut.cpp @@ -275,13 +275,13 @@ Y_UNIT_TEST_SUITE(TLzTest) { UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789"); } - - Y_UNIT_TEST(TestYQ609) { - auto data = NResource::Find("/yq_609.data"); - - TMemoryInput input(data.Data(), data.Size()); - - TLz4Decompress d(&input); - UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError); - } + + Y_UNIT_TEST(TestYQ609) { + auto data = NResource::Find("/yq_609.data"); + + TMemoryInput input(data.Data(), data.Size()); + + TLz4Decompress d(&input); + UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError); + } } diff --git a/library/cpp/streams/lz/ut/ya.make b/library/cpp/streams/lz/ut/ya.make index ca35b53f101..18288c8ac91 100644 --- a/library/cpp/streams/lz/ut/ya.make +++ b/library/cpp/streams/lz/ut/ya.make @@ -8,7 +8,7 @@ OWNER( RESOURCE( random.data /random.data request.data /request.data - yq_609.data /yq_609.data + yq_609.data /yq_609.data ) SRCS( diff --git a/library/cpp/streams/lz/ut/yq_609.data b/library/cpp/streams/lz/ut/yq_609.data Binary files differindex 144f4b4cc70..9694b0c97f1 100644 --- a/library/cpp/streams/lz/ut/yq_609.data +++ b/library/cpp/streams/lz/ut/yq_609.data diff --git a/ydb/core/driver_lib/run/factories.h b/ydb/core/driver_lib/run/factories.h index 92ad3f8b5e2..41a45b44e08 100644 --- a/ydb/core/driver_lib/run/factories.h +++ b/ydb/core/driver_lib/run/factories.h @@ -19,7 +19,7 @@ #include <library/cpp/actors/core/actorsystem.h> #include <ydb/library/security/ydb_credentials_provider_factory.h> - + #include <functional> #include <unordered_map> #include <unordered_set> @@ -46,7 +46,7 @@ struct TModuleFactories { IActor*(*FolderServiceFactory)(const NKikimrProto::NFolderService::TFolderServiceConfig&); std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)> YqAuditServiceFactory; - NKikimr::TYdbCredentialsProviderFactory YdbCredentialProviderFactory; + NKikimr::TYdbCredentialsProviderFactory YdbCredentialProviderFactory; // Factory for grpc services TGrpcServiceFactory GrpcServiceFactory; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 905a0e7027a..819c1478d19 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -182,8 +182,8 @@ #include <util/digest/city.h> #include <util/generic/algorithm.h> -#include <util/generic/size_literals.h> - +#include <util/generic/size_literals.h> + #include <util/system/hostname.h> namespace NKikimr { diff --git a/ydb/core/grpc_services/rpc_yq.cpp b/ydb/core/grpc_services/rpc_yq.cpp index eca0203857b..6fe5f868d6b 100644 --- a/ydb/core/grpc_services/rpc_yq.cpp +++ b/ydb/core/grpc_services/rpc_yq.cpp @@ -35,13 +35,13 @@ public: using TBase::GetProtoRequest; protected: - TString Token; - TString FolderId; - TString User; - TString PeerName; - TString UserAgent; - TString RequestId; - + TString Token; + TString FolderId; + TString User; + TString PeerName; + TString UserAgent; + TString RequestId; + public: TYandexQueryRequestRPC(IRequestOpCtx* request) : TBase(request) {} @@ -51,7 +51,7 @@ public: auto request = dynamic_cast<RpcRequestType*>(requestCtx); Y_VERIFY(request); - + auto proxyCtx = dynamic_cast<IRequestProxyCtx*>(requestCtx); Y_VERIFY(proxyCtx); @@ -78,13 +78,13 @@ public: return; } - FolderId = path.back(); - if (!FolderId) { + FolderId = path.back(); + if (!FolderId) { ReplyWithStatus("Folder id is empty", StatusIds::BAD_REQUEST); return; } - if (FolderId.length() > 1024) { + if (FolderId.length() > 1024) { ReplyWithStatus("Folder id length greater than 1024 characters: " + FolderId, StatusIds::BAD_REQUEST); return; } @@ -93,7 +93,7 @@ public: TVector<TString> permissions; if (internalToken) { NACLib::TUserToken userToken(internalToken); - User = userToken.GetUserSID(); + User = userToken.GetUserSID(); for (const auto& sid: request->Sids) { if (userToken.IsExist(sid)) { permissions.push_back(sid); @@ -101,7 +101,7 @@ public: } } - if (!User) { + if (!User) { ReplyWithStatus("Authorization error. Permission denied", StatusIds::UNAUTHORIZED); return; } @@ -142,20 +142,20 @@ protected: req.SendResult(response.Result, StatusIds::SUCCESS); } - NYq::TEvAuditService::TExtraInfo extraInfo{ - .Token = Token, - .FolderId = FolderId, - .User = User, - .PeerName = PeerName, - .UserAgent = UserAgent, - .RequestId = RequestId, - }; - - Send(NYq::YqAuditServiceActorId(), NYq::TEvAuditService::MakeAuditEvent( - std::move(extraInfo), - *GetProtoRequest(), - response.Issues, - response.AuditDetails)); + NYq::TEvAuditService::TExtraInfo extraInfo{ + .Token = Token, + .FolderId = FolderId, + .User = User, + .PeerName = PeerName, + .UserAgent = UserAgent, + .RequestId = RequestId, + }; + + Send(NYq::YqAuditServiceActorId(), NYq::TEvAuditService::MakeAuditEvent( + std::move(extraInfo), + *GetProtoRequest(), + response.Issues, + response.AuditDetails)); } void Handle(typename EvResponseType::TPtr& ev) { diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 52b4ca916a6..d4907c26f1e 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -723,7 +723,7 @@ namespace Tests { auto& controlPlaneProxyConfig = *protoConfig.MutableControlPlaneProxy(); controlPlaneProxyConfig.SetEnabled(true); } - + { auto& testConnectionConfig = *protoConfig.MutableTestConnection(); testConnectionConfig.SetEnabled(true); @@ -752,7 +752,7 @@ namespace Tests { commonConfig.SetYdbMvpCloudEndpoint(ydbMvpEndpoint); commonConfig.SetIdsPrefix("ut"); } - + { auto& privateApiConfig = *protoConfig.MutablePrivateApi(); privateApiConfig.SetEnabled(true); @@ -800,7 +800,7 @@ namespace Tests { Runtime->RegisterService(serviceActorId, actorId, nodeIdx); }; - const auto ydbCredFactory = NKikimr::CreateYdbCredentialsProviderFactory; + const auto ydbCredFactory = NKikimr::CreateYdbCredentialsProviderFactory; auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); auto yqSharedResources = NYq::CreateYqSharedResources(protoConfig, ydbCredFactory, counters); NYq::Init( @@ -811,8 +811,8 @@ namespace Tests { "TestTenant", nullptr, // MakeIntrusive<NPq::NConfigurationManager::TConnections>(), yqSharedResources, - NKikimr::NFolderService::CreateMockFolderServiceActor, - NYq::CreateMockYqAuditServiceActor, + NKikimr::NFolderService::CreateMockFolderServiceActor, + NYq::CreateMockYqAuditServiceActor, ydbCredFactory, /*IcPort = */0 ); diff --git a/ydb/core/yq/libs/actors/clusters_from_connections.cpp b/ydb/core/yq/libs/actors/clusters_from_connections.cpp index 50b22babc74..93e04718927 100644 --- a/ydb/core/yq/libs/actors/clusters_from_connections.cpp +++ b/ydb/core/yq/libs/actors/clusters_from_connections.cpp @@ -111,12 +111,12 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection> if (solomonEndpoint.empty()) { if (connectionName.StartsWith("pre")) { solomonEndpoint = "monitoring.api.cloud-preprod.yandex.net"; - clusterCfg->SetUseSsl(true); + clusterCfg->SetUseSsl(true); } else if (connectionName.StartsWith("so")) { solomonEndpoint = "solomon.yandex.net"; } else { solomonEndpoint = "monitoring.api.cloud.yandex.net"; - clusterCfg->SetUseSsl(true); + clusterCfg->SetUseSsl(true); } } diff --git a/ydb/core/yq/libs/actors/run_actor.cpp b/ydb/core/yq/libs/actors/run_actor.cpp index 14eeaf2e611..5549f8f254e 100644 --- a/ydb/core/yq/libs/actors/run_actor.cpp +++ b/ydb/core/yq/libs/actors/run_actor.cpp @@ -856,7 +856,7 @@ private: TDqConfiguration::TPtr dqConfiguration = MakeIntrusive<TDqConfiguration>(); dqConfiguration->Dispatch(dqGraphParams.GetSettings()); dqConfiguration->FreezeDefaults(); - dqConfiguration->FallbackPolicy = "never"; + dqConfiguration->FallbackPolicy = "never"; ExecuterId = NActors::TActivationContext::Register(NYql::NDq::MakeDqExecuter(MakeYqlNodesManagerId(), SelfId(), Params.QueryId, "", dqConfiguration, ServiceCounters.Counters, TInstant::Now(), EnableCheckpointCoordinator)); @@ -937,12 +937,12 @@ private: attr->SetName("_OneGraphPerQuery"); attr->SetValue("1"); } - - if (Params.QueryType == YandexQuery::QueryContent_QueryType_ANALYTICS) { - attr = dqSettings.Add(); - attr->SetName("AnalyticsHopping"); - attr->SetValue("1"); - } + + if (Params.QueryType == YandexQuery::QueryContent_QueryType_ANALYTICS) { + attr = dqSettings.Add(); + attr->SetName("AnalyticsHopping"); + attr->SetValue("1"); + } } void AddClustersFromConfig(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters) const { @@ -1163,12 +1163,12 @@ private: dataProvidersInit.push_back(GetPqDataProviderInitializer(pqGateway, false, dbResolver)); } - { - auto solomonConfig = gatewaysConfig.GetSolomon(); - auto solomonGateway = NYql::CreateSolomonGateway(solomonConfig); - dataProvidersInit.push_back(GetSolomonDataProviderInitializer(solomonGateway, false)); - } - + { + auto solomonConfig = gatewaysConfig.GetSolomon(); + auto solomonGateway = NYql::CreateSolomonGateway(solomonConfig); + dataProvidersInit.push_back(GetSolomonDataProviderInitializer(solomonGateway, false)); + } + TProgramFactory progFactory(false, Params.FunctionRegistry, Params.NextUniqueId, dataProvidersInit, "yq"); progFactory.SetModules(Params.ModuleResolver); progFactory.SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(Params.FunctionRegistry, nullptr)); diff --git a/ydb/core/yq/libs/actors/system_clusters.cpp b/ydb/core/yq/libs/actors/system_clusters.cpp index bfa8eb99020..2968ae64872 100644 --- a/ydb/core/yq/libs/actors/system_clusters.cpp +++ b/ydb/core/yq/libs/actors/system_clusters.cpp @@ -19,16 +19,16 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin } { const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); - clusterCfg->SetName("logbroker_iam_no_sa"); - clusterCfg->SetEndpoint("logbroker.yandex.net:2135"); - clusterCfg->SetConfigManagerEndpoint("cm.logbroker.yandex.net:1111"); - clusterCfg->SetAddBearerToToken(true); - clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); - clusterCfg->SetToken(authToken); - clusters.emplace(clusterCfg->GetName(), PqProviderName); - } - { - const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); + clusterCfg->SetName("logbroker_iam_no_sa"); + clusterCfg->SetEndpoint("logbroker.yandex.net:2135"); + clusterCfg->SetConfigManagerEndpoint("cm.logbroker.yandex.net:1111"); + clusterCfg->SetAddBearerToToken(true); + clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); + clusterCfg->SetToken(authToken); + clusters.emplace(clusterCfg->GetName(), PqProviderName); + } + { + const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping(); clusterCfg->SetName("lbkx"); clusterCfg->SetEndpoint("lbkx.logbroker.yandex.net:2135"); clusterCfg->SetConfigManagerEndpoint("cm.lbkx.logbroker.yandex.net:1111"); @@ -89,7 +89,7 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin clusterCfg->SetEndpoint("lb.cc8035oc71oh9um52mv3.ydb.mdb.cloud-preprod.yandex.net:2135"); clusterCfg->SetConfigManagerEndpoint("cm.global.logbroker.cloud-preprod.yandex.net:1111"); clusterCfg->SetDatabase("/pre-prod_global/aoeb66ftj1tbt1b2eimn/cc8035oc71oh9um52mv3"); - clusterCfg->SetUseSsl(true); + clusterCfg->SetUseSsl(true); clusterCfg->SetServiceAccountId("bfbf4nc8c849ej11k3aq"); // audit-trails-reader-sa clusterCfg->SetServiceAccountIdSignature("SyFk1SpKdQa6L8IJxnwvgC57yMQ="); clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); @@ -101,7 +101,7 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin clusterCfg->SetEndpoint("lb.etn03iai600jur7pipla.ydb.mdb.yandexcloud.net:2135"); clusterCfg->SetConfigManagerEndpoint("cm.global.logbroker.cloud.yandex.net:1111"); clusterCfg->SetDatabase("/global/b1gvcqr959dbmi1jltep/etn03iai600jur7pipla"); - clusterCfg->SetUseSsl(true); + clusterCfg->SetUseSsl(true); clusterCfg->SetServiceAccountId("aje0ir9755dd4ia9mpu5"); clusterCfg->SetServiceAccountIdSignature("xxx"); clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE); diff --git a/ydb/core/yq/libs/audit/events/events.cpp b/ydb/core/yq/libs/audit/events/events.cpp index a4930159cf2..6c3d2603e7e 100644 --- a/ydb/core/yq/libs/audit/events/events.cpp +++ b/ydb/core/yq/libs/audit/events/events.cpp @@ -1 +1 @@ -#include "events.h" +#include "events.h" diff --git a/ydb/core/yq/libs/audit/events/events.h b/ydb/core/yq/libs/audit/events/events.h index 75cf23da197..15026dc082a 100644 --- a/ydb/core/yq/libs/audit/events/events.h +++ b/ydb/core/yq/libs/audit/events/events.h @@ -1,186 +1,186 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/issue/yql_issue.h> - + #include <ydb/public/api/protos/yq.pb.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/checkpointing_common/defs.h> - -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/event_pb.h> -#include <library/cpp/actors/interconnect/events_local.h> - -#include <optional> - -namespace NYq { - -struct TEvAuditService { - struct TExtraInfo { - TString Token; - TString CloudId; - TString FolderId; - TString User; - - TString PeerName; - TString UserAgent; - TString RequestId; - }; - - // Event ids. - enum EEv : ui32 { + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/event_pb.h> +#include <library/cpp/actors/interconnect/events_local.h> + +#include <optional> + +namespace NYq { + +struct TEvAuditService { + struct TExtraInfo { + TString Token; + TString CloudId; + TString FolderId; + TString User; + + TString PeerName; + TString UserAgent; + TString RequestId; + }; + + // Event ids. + enum EEv : ui32 { EvCreateBindingReport = YqEventSubspaceBegin(NYq::TYqEventSubspace::AuditService), - EvModifyBindingReport, - EvDeleteBindingReport, - EvCreateConnectionReport, - EvModifyConnectionReport, - EvDeleteConnectionReport, - EvCreateQueryReport, - EvControlQueryReport, - EvModifyQueryReport, - EvDeleteQueryReport, - EvEnd, - }; - + EvModifyBindingReport, + EvDeleteBindingReport, + EvCreateConnectionReport, + EvModifyConnectionReport, + EvDeleteConnectionReport, + EvCreateQueryReport, + EvControlQueryReport, + EvModifyQueryReport, + EvDeleteQueryReport, + EvEnd, + }; + static_assert(EvEnd <= YqEventSubspaceEnd(NYq::TYqEventSubspace::AuditService), "All events must be in their subspace"); - -private: - template <class TRequest, class TAuditDetailsObj> - struct TAuditReportBase { - explicit TAuditReportBase( - TExtraInfo&& extraInfo, - const TRequest& request, - const NYql::TIssues& issues, - const TAuditDetails<TAuditDetailsObj>& details, - std::optional<TString> eventId = std::nullopt) - : ExtraInfo(std::move(extraInfo)) - , Request(request) - , Issues(issues) - , Details(details) - , EventId(eventId) - { - } - - TExtraInfo ExtraInfo; - TRequest Request; - NYql::TIssues Issues; - TAuditDetails<TAuditDetailsObj> Details; - std::optional<TString> EventId; - }; - -public: - template <class TRequest, class TAuditDetailsObj> - struct TAuditReport; - - // CreateBinding - using CreateBindingAuditReport = TAuditReport<YandexQuery::CreateBindingRequest, YandexQuery::Binding>; - - template<> - struct TAuditReport<YandexQuery::CreateBindingRequest, YandexQuery::Binding> - : NActors::TEventLocal<CreateBindingAuditReport, EvCreateBindingReport> - , TAuditReportBase<YandexQuery::CreateBindingRequest, YandexQuery::Binding> { - using TAuditReportBase::TAuditReportBase; - }; - - // ModifyBinding - using ModifyBindingAuditReport = TAuditReport<YandexQuery::ModifyBindingRequest, YandexQuery::Binding>; - - template<> - struct TAuditReport<YandexQuery::ModifyBindingRequest, YandexQuery::Binding> - : NActors::TEventLocal<ModifyBindingAuditReport, EvModifyBindingReport> - , TAuditReportBase<YandexQuery::ModifyBindingRequest, YandexQuery::Binding> { - using TAuditReportBase::TAuditReportBase; - }; - - // DeleteBinding - using DeleteBindingAuditReport = TAuditReport<YandexQuery::DeleteBindingRequest, YandexQuery::Binding>; - - template<> - struct TAuditReport<YandexQuery::DeleteBindingRequest, YandexQuery::Binding> - : NActors::TEventLocal<DeleteBindingAuditReport, EvDeleteBindingReport> - , TAuditReportBase<YandexQuery::DeleteBindingRequest, YandexQuery::Binding> { - using TAuditReportBase::TAuditReportBase; - }; - - // CreateConnection - using CreateConnectionAuditReport = TAuditReport<YandexQuery::CreateConnectionRequest, YandexQuery::Connection>; - - template<> - struct TAuditReport<YandexQuery::CreateConnectionRequest, YandexQuery::Connection> - : NActors::TEventLocal<CreateConnectionAuditReport, EvCreateConnectionReport> - , TAuditReportBase<YandexQuery::CreateConnectionRequest, YandexQuery::Connection> { - using TAuditReportBase::TAuditReportBase; - }; - - // ModifyConnection - using ModifyConnectionAuditReport = TAuditReport<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection>; - - template<> - struct TAuditReport<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection> - : NActors::TEventLocal<ModifyConnectionAuditReport, EvModifyConnectionReport> - , TAuditReportBase<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection> { - using TAuditReportBase::TAuditReportBase; - }; - - // DeleteConnection - using DeleteConnectionAuditReport = TAuditReport<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection>; - - template<> - struct TAuditReport<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection> - : NActors::TEventLocal<DeleteConnectionAuditReport, EvDeleteConnectionReport> - , TAuditReportBase<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection> { - using TAuditReportBase::TAuditReportBase; - }; - - // CreateQuery - using CreateQueryAuditReport = TAuditReport<YandexQuery::CreateQueryRequest, YandexQuery::Query>; - - template<> - struct TAuditReport<YandexQuery::CreateQueryRequest, YandexQuery::Query> - : NActors::TEventLocal<CreateQueryAuditReport, EvCreateQueryReport> - , TAuditReportBase<YandexQuery::CreateQueryRequest, YandexQuery::Query> { - using TAuditReportBase::TAuditReportBase; - }; - - // ControlQuery - using ControlQueryAuditReport = TAuditReport<YandexQuery::ControlQueryRequest, YandexQuery::Query>; - - template<> - struct TAuditReport<YandexQuery::ControlQueryRequest, YandexQuery::Query> - : NActors::TEventLocal<ControlQueryAuditReport, EvControlQueryReport> - , TAuditReportBase<YandexQuery::ControlQueryRequest, YandexQuery::Query> { - using TAuditReportBase::TAuditReportBase; - }; - - // ModifyQuery - using ModifyQueryAuditReport = TAuditReport<YandexQuery::ModifyQueryRequest, YandexQuery::Query>; - - template<> - struct TAuditReport<YandexQuery::ModifyQueryRequest, YandexQuery::Query> - : NActors::TEventLocal<ModifyQueryAuditReport, EvModifyQueryReport> - , TAuditReportBase<YandexQuery::ModifyQueryRequest, YandexQuery::Query> { - using TAuditReportBase::TAuditReportBase; - }; - - // DeleteQuery - using DeleteQueryAuditReport = TAuditReport<YandexQuery::DeleteQueryRequest, YandexQuery::Query>; - - template<> - struct TAuditReport<YandexQuery::DeleteQueryRequest, YandexQuery::Query> - : NActors::TEventLocal<DeleteQueryAuditReport, EvDeleteQueryReport> - , TAuditReportBase<YandexQuery::DeleteQueryRequest, YandexQuery::Query> { - using TAuditReportBase::TAuditReportBase; - }; - - - template <class TRequest, class TAuditDetailsObj> - static TAuditReport<TRequest, TAuditDetailsObj>* MakeAuditEvent( - TExtraInfo&& extraInfo, - const TRequest& request, - const NYql::TIssues& issues, - const TAuditDetails<TAuditDetailsObj>& details, - std::optional<TString> eventId = std::nullopt) { - return new TAuditReport<TRequest, TAuditDetailsObj>(std::move(extraInfo), request, issues, details, eventId); - } -}; - -} // namespace NYq + +private: + template <class TRequest, class TAuditDetailsObj> + struct TAuditReportBase { + explicit TAuditReportBase( + TExtraInfo&& extraInfo, + const TRequest& request, + const NYql::TIssues& issues, + const TAuditDetails<TAuditDetailsObj>& details, + std::optional<TString> eventId = std::nullopt) + : ExtraInfo(std::move(extraInfo)) + , Request(request) + , Issues(issues) + , Details(details) + , EventId(eventId) + { + } + + TExtraInfo ExtraInfo; + TRequest Request; + NYql::TIssues Issues; + TAuditDetails<TAuditDetailsObj> Details; + std::optional<TString> EventId; + }; + +public: + template <class TRequest, class TAuditDetailsObj> + struct TAuditReport; + + // CreateBinding + using CreateBindingAuditReport = TAuditReport<YandexQuery::CreateBindingRequest, YandexQuery::Binding>; + + template<> + struct TAuditReport<YandexQuery::CreateBindingRequest, YandexQuery::Binding> + : NActors::TEventLocal<CreateBindingAuditReport, EvCreateBindingReport> + , TAuditReportBase<YandexQuery::CreateBindingRequest, YandexQuery::Binding> { + using TAuditReportBase::TAuditReportBase; + }; + + // ModifyBinding + using ModifyBindingAuditReport = TAuditReport<YandexQuery::ModifyBindingRequest, YandexQuery::Binding>; + + template<> + struct TAuditReport<YandexQuery::ModifyBindingRequest, YandexQuery::Binding> + : NActors::TEventLocal<ModifyBindingAuditReport, EvModifyBindingReport> + , TAuditReportBase<YandexQuery::ModifyBindingRequest, YandexQuery::Binding> { + using TAuditReportBase::TAuditReportBase; + }; + + // DeleteBinding + using DeleteBindingAuditReport = TAuditReport<YandexQuery::DeleteBindingRequest, YandexQuery::Binding>; + + template<> + struct TAuditReport<YandexQuery::DeleteBindingRequest, YandexQuery::Binding> + : NActors::TEventLocal<DeleteBindingAuditReport, EvDeleteBindingReport> + , TAuditReportBase<YandexQuery::DeleteBindingRequest, YandexQuery::Binding> { + using TAuditReportBase::TAuditReportBase; + }; + + // CreateConnection + using CreateConnectionAuditReport = TAuditReport<YandexQuery::CreateConnectionRequest, YandexQuery::Connection>; + + template<> + struct TAuditReport<YandexQuery::CreateConnectionRequest, YandexQuery::Connection> + : NActors::TEventLocal<CreateConnectionAuditReport, EvCreateConnectionReport> + , TAuditReportBase<YandexQuery::CreateConnectionRequest, YandexQuery::Connection> { + using TAuditReportBase::TAuditReportBase; + }; + + // ModifyConnection + using ModifyConnectionAuditReport = TAuditReport<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection>; + + template<> + struct TAuditReport<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection> + : NActors::TEventLocal<ModifyConnectionAuditReport, EvModifyConnectionReport> + , TAuditReportBase<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection> { + using TAuditReportBase::TAuditReportBase; + }; + + // DeleteConnection + using DeleteConnectionAuditReport = TAuditReport<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection>; + + template<> + struct TAuditReport<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection> + : NActors::TEventLocal<DeleteConnectionAuditReport, EvDeleteConnectionReport> + , TAuditReportBase<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection> { + using TAuditReportBase::TAuditReportBase; + }; + + // CreateQuery + using CreateQueryAuditReport = TAuditReport<YandexQuery::CreateQueryRequest, YandexQuery::Query>; + + template<> + struct TAuditReport<YandexQuery::CreateQueryRequest, YandexQuery::Query> + : NActors::TEventLocal<CreateQueryAuditReport, EvCreateQueryReport> + , TAuditReportBase<YandexQuery::CreateQueryRequest, YandexQuery::Query> { + using TAuditReportBase::TAuditReportBase; + }; + + // ControlQuery + using ControlQueryAuditReport = TAuditReport<YandexQuery::ControlQueryRequest, YandexQuery::Query>; + + template<> + struct TAuditReport<YandexQuery::ControlQueryRequest, YandexQuery::Query> + : NActors::TEventLocal<ControlQueryAuditReport, EvControlQueryReport> + , TAuditReportBase<YandexQuery::ControlQueryRequest, YandexQuery::Query> { + using TAuditReportBase::TAuditReportBase; + }; + + // ModifyQuery + using ModifyQueryAuditReport = TAuditReport<YandexQuery::ModifyQueryRequest, YandexQuery::Query>; + + template<> + struct TAuditReport<YandexQuery::ModifyQueryRequest, YandexQuery::Query> + : NActors::TEventLocal<ModifyQueryAuditReport, EvModifyQueryReport> + , TAuditReportBase<YandexQuery::ModifyQueryRequest, YandexQuery::Query> { + using TAuditReportBase::TAuditReportBase; + }; + + // DeleteQuery + using DeleteQueryAuditReport = TAuditReport<YandexQuery::DeleteQueryRequest, YandexQuery::Query>; + + template<> + struct TAuditReport<YandexQuery::DeleteQueryRequest, YandexQuery::Query> + : NActors::TEventLocal<DeleteQueryAuditReport, EvDeleteQueryReport> + , TAuditReportBase<YandexQuery::DeleteQueryRequest, YandexQuery::Query> { + using TAuditReportBase::TAuditReportBase; + }; + + + template <class TRequest, class TAuditDetailsObj> + static TAuditReport<TRequest, TAuditDetailsObj>* MakeAuditEvent( + TExtraInfo&& extraInfo, + const TRequest& request, + const NYql::TIssues& issues, + const TAuditDetails<TAuditDetailsObj>& details, + std::optional<TString> eventId = std::nullopt) { + return new TAuditReport<TRequest, TAuditDetailsObj>(std::move(extraInfo), request, issues, details, eventId); + } +}; + +} // namespace NYq diff --git a/ydb/core/yq/libs/audit/events/ya.make b/ydb/core/yq/libs/audit/events/ya.make index 44fb815a370..cb36076615b 100644 --- a/ydb/core/yq/libs/audit/events/ya.make +++ b/ydb/core/yq/libs/audit/events/ya.make @@ -1,18 +1,18 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - events.cpp -) - -PEERDIR( - library/cpp/actors/core - library/cpp/actors/interconnect + +LIBRARY() + +SRCS( + events.cpp +) + +PEERDIR( + library/cpp/actors/core + library/cpp/actors/interconnect ydb/core/yq/libs/checkpointing_common ydb/core/yq/libs/control_plane_storage/events ydb/public/api/protos ydb/library/yql/public/issue -) - -END() +) + +END() diff --git a/ydb/core/yq/libs/audit/mock/ya.make b/ydb/core/yq/libs/audit/mock/ya.make index 3f7e5f017d7..2d8dc3ebe5d 100644 --- a/ydb/core/yq/libs/audit/mock/ya.make +++ b/ydb/core/yq/libs/audit/mock/ya.make @@ -1,14 +1,14 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - yq_mock_audit_service.cpp -) - -PEERDIR( + +LIBRARY() + +SRCS( + yq_mock_audit_service.cpp +) + +PEERDIR( ydb/core/yq/libs/audit/events ydb/core/yq/libs/config/protos -) - -END() +) + +END() diff --git a/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp b/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp index cbe5a91b272..74f994e0df9 100644 --- a/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp +++ b/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp @@ -1,75 +1,75 @@ -#include "yq_mock_audit_service.h" - +#include "yq_mock_audit_service.h" + #include <ydb/core/yq/libs/audit/events/events.h> - -#include <library/cpp/actors/core/hfunc.h> - -namespace NYq { - -class TYqMockAuditServiceActor : public NActors::TActor<TYqMockAuditServiceActor> { -public: - TYqMockAuditServiceActor() : TActor<TYqMockAuditServiceActor>(&TYqMockAuditServiceActor::StateFunc) {} - + +#include <library/cpp/actors/core/hfunc.h> + +namespace NYq { + +class TYqMockAuditServiceActor : public NActors::TActor<TYqMockAuditServiceActor> { +public: + TYqMockAuditServiceActor() : TActor<TYqMockAuditServiceActor>(&TYqMockAuditServiceActor::StateFunc) {} + static constexpr char ActorName[] = "YQ_MOCK_AUDIT_SERVICE"; -private: - STRICT_STFUNC(StateFunc, - hFunc(TEvAuditService::CreateBindingAuditReport, Handle); - hFunc(TEvAuditService::ModifyBindingAuditReport, Handle); - hFunc(TEvAuditService::DeleteBindingAuditReport, Handle); - hFunc(TEvAuditService::CreateConnectionAuditReport, Handle); - hFunc(TEvAuditService::ModifyConnectionAuditReport, Handle); - hFunc(TEvAuditService::DeleteConnectionAuditReport, Handle); - hFunc(TEvAuditService::CreateQueryAuditReport, Handle); - hFunc(TEvAuditService::ControlQueryAuditReport, Handle); - hFunc(TEvAuditService::ModifyQueryAuditReport, Handle); - hFunc(TEvAuditService::DeleteQueryAuditReport, Handle); - ) - - void Handle(TEvAuditService::CreateBindingAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::ModifyBindingAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::DeleteBindingAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::CreateConnectionAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::ModifyConnectionAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::DeleteConnectionAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::CreateQueryAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::ControlQueryAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::ModifyQueryAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } - - void Handle(TEvAuditService::DeleteQueryAuditReport::TPtr& ev) { - Y_UNUSED(ev); - } -}; - +private: + STRICT_STFUNC(StateFunc, + hFunc(TEvAuditService::CreateBindingAuditReport, Handle); + hFunc(TEvAuditService::ModifyBindingAuditReport, Handle); + hFunc(TEvAuditService::DeleteBindingAuditReport, Handle); + hFunc(TEvAuditService::CreateConnectionAuditReport, Handle); + hFunc(TEvAuditService::ModifyConnectionAuditReport, Handle); + hFunc(TEvAuditService::DeleteConnectionAuditReport, Handle); + hFunc(TEvAuditService::CreateQueryAuditReport, Handle); + hFunc(TEvAuditService::ControlQueryAuditReport, Handle); + hFunc(TEvAuditService::ModifyQueryAuditReport, Handle); + hFunc(TEvAuditService::DeleteQueryAuditReport, Handle); + ) + + void Handle(TEvAuditService::CreateBindingAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::ModifyBindingAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::DeleteBindingAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::CreateConnectionAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::ModifyConnectionAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::DeleteConnectionAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::CreateQueryAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::ControlQueryAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::ModifyQueryAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } + + void Handle(TEvAuditService::DeleteQueryAuditReport::TPtr& ev) { + Y_UNUSED(ev); + } +}; + NActors::IActor* CreateMockYqAuditServiceActor(const NConfig::TAuditConfig& config) { - Y_UNUSED(config); - return new TYqMockAuditServiceActor(); -} - -} // namespace NYq + Y_UNUSED(config); + return new TYqMockAuditServiceActor(); +} + +} // namespace NYq diff --git a/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h b/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h index 27c345420a5..f00bf089599 100644 --- a/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h +++ b/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h @@ -1,14 +1,14 @@ -#pragma once - +#pragma once + #include <ydb/core/yq/libs/config/protos/audit.pb.h> - -#include <library/cpp/actors/core/actor.h> - -#include <util/system/types.h> - - -namespace NYq { - + +#include <library/cpp/actors/core/actor.h> + +#include <util/system/types.h> + + +namespace NYq { + NActors::IActor* CreateMockYqAuditServiceActor(const NConfig::TAuditConfig& config); - -} // namespace NYq + +} // namespace NYq diff --git a/ydb/core/yq/libs/audit/ya.make b/ydb/core/yq/libs/audit/ya.make index 947c8873bc1..a0671f84d47 100644 --- a/ydb/core/yq/libs/audit/ya.make +++ b/ydb/core/yq/libs/audit/ya.make @@ -1,14 +1,14 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - yq_audit_service.cpp -) - -END() - -RECURSE( - events - mock -) + +LIBRARY() + +SRCS( + yq_audit_service.cpp +) + +END() + +RECURSE( + events + mock +) diff --git a/ydb/core/yq/libs/audit/yq_audit_service.cpp b/ydb/core/yq/libs/audit/yq_audit_service.cpp index 910422f276f..7e9be493360 100644 --- a/ydb/core/yq/libs/audit/yq_audit_service.cpp +++ b/ydb/core/yq/libs/audit/yq_audit_service.cpp @@ -1,13 +1,13 @@ -#include <library/cpp/actors/core/actor.h> - -#include <util/system/types.h> - - -namespace NYq { - -NActors::TActorId YqAuditServiceActorId() { - constexpr TStringBuf name = "YQAUDSVC"; - return NActors::TActorId(0, name); -} - -} // namespace NYq +#include <library/cpp/actors/core/actor.h> + +#include <util/system/types.h> + + +namespace NYq { + +NActors::TActorId YqAuditServiceActorId() { + constexpr TStringBuf name = "YQAUDSVC"; + return NActors::TActorId(0, name); +} + +} // namespace NYq diff --git a/ydb/core/yq/libs/audit/yq_audit_service.h b/ydb/core/yq/libs/audit/yq_audit_service.h index 150938657a5..c750c4d09c0 100644 --- a/ydb/core/yq/libs/audit/yq_audit_service.h +++ b/ydb/core/yq/libs/audit/yq_audit_service.h @@ -1,12 +1,12 @@ -#pragma once - -#include <library/cpp/actors/core/actor.h> - -#include <util/system/types.h> - - -namespace NYq { - -NActors::TActorId YqAuditServiceActorId(); - -} // namespace NYq +#pragma once + +#include <library/cpp/actors/core/actor.h> + +#include <util/system/types.h> + + +namespace NYq { + +NActors::TActorId YqAuditServiceActorId(); + +} // namespace NYq diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp index 220a37f233b..07c8d5101a0 100644 --- a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp +++ b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp @@ -40,13 +40,13 @@ class TStorageProxy : public TActorBootstrapped<TStorageProxy> { TCheckpointStoragePtr CheckpointStorage; TStateStoragePtr StateStorage; TActorId ActorGC; - NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory; + NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory; public: - explicit TStorageProxy( + explicit TStorageProxy( const NConfig::TCheckpointCoordinatorConfig& config, const NConfig::TCommonConfig& commonConfig, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); void Bootstrap(); @@ -97,10 +97,10 @@ static void FillDefaultParameters(NConfig::TCheckpointCoordinatorConfig& checkpo } } -TStorageProxy::TStorageProxy( +TStorageProxy::TStorageProxy( const NConfig::TCheckpointCoordinatorConfig& config, const NConfig::TCommonConfig& commonConfig, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) : Config(config) , CommonConfig(commonConfig) , StorageConfig(Config.GetStorage()) @@ -390,11 +390,11 @@ void TStorageProxy::Handle(NYql::NDq::TEvDqCompute::TEvGetTaskState::TPtr& ev) { //////////////////////////////////////////////////////////////////////////////// -std::unique_ptr<NActors::IActor> NewStorageProxy( +std::unique_ptr<NActors::IActor> NewStorageProxy( const NConfig::TCheckpointCoordinatorConfig& config, const NConfig::TCommonConfig& commonConfig, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) -{ + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) +{ return std::unique_ptr<NActors::IActor>(new TStorageProxy(config, commonConfig, credentialsProviderFactory)); } diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.h b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.h index f1b368f96f1..c67d928a151 100644 --- a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.h +++ b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.h @@ -4,16 +4,16 @@ #include <ydb/core/yq/libs/config/protos/common.pb.h> #include <ydb/library/security/ydb_credentials_provider_factory.h> - + #include <library/cpp/actors/core/actor.h> #include <memory> namespace NYq { -std::unique_ptr<NActors::IActor> NewStorageProxy( +std::unique_ptr<NActors::IActor> NewStorageProxy( const NConfig::TCheckpointCoordinatorConfig& config, const NConfig::TCommonConfig& commonConfig, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); } // namespace NYq diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_service.cpp b/ydb/core/yq/libs/checkpoint_storage/storage_service.cpp index b6825d8ef9f..421c19d7fdb 100644 --- a/ydb/core/yq/libs/checkpoint_storage/storage_service.cpp +++ b/ydb/core/yq/libs/checkpoint_storage/storage_service.cpp @@ -8,11 +8,11 @@ using namespace NActors; //////////////////////////////////////////////////////////////////////////////// -std::unique_ptr<NActors::IActor> NewCheckpointStorageService( +std::unique_ptr<NActors::IActor> NewCheckpointStorageService( const NConfig::TCheckpointCoordinatorConfig& config, const NConfig::TCommonConfig& commonConfig, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) -{ + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) +{ return NewStorageProxy(config, commonConfig, credentialsProviderFactory); } diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_service.h b/ydb/core/yq/libs/checkpoint_storage/storage_service.h index 53f7cb8c430..5eab844545a 100644 --- a/ydb/core/yq/libs/checkpoint_storage/storage_service.h +++ b/ydb/core/yq/libs/checkpoint_storage/storage_service.h @@ -4,16 +4,16 @@ #include <ydb/core/yq/libs/config/protos/common.pb.h> #include <ydb/library/security/ydb_credentials_provider_factory.h> - + #include <library/cpp/actors/core/actor.h> #include <memory> namespace NYq { -std::unique_ptr<NActors::IActor> NewCheckpointStorageService( +std::unique_ptr<NActors::IActor> NewCheckpointStorageService( const NConfig::TCheckpointCoordinatorConfig& config, const NConfig::TCommonConfig& commonConfig, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); } // namespace NYq diff --git a/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp b/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp index 2e8acabc841..7799b52daaf 100644 --- a/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp +++ b/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp @@ -552,8 +552,8 @@ class TCheckpointStorage : public ICheckpointStorage { const NConfig::TYdbStorageConfig Config; public: - explicit TCheckpointStorage( - const NConfig::TYdbStorageConfig& config, + explicit TCheckpointStorage( + const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const IEntityIdGenerator::TPtr& entityIdGenerator); @@ -621,11 +621,11 @@ private: //////////////////////////////////////////////////////////////////////////////// -TCheckpointStorage::TCheckpointStorage( - const NConfig::TYdbStorageConfig& config, +TCheckpointStorage::TCheckpointStorage( + const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const IEntityIdGenerator::TPtr& entityIdGenerator) - : YdbConnection(NewYdbConnection(config, credentialsProviderFactory)) + : YdbConnection(NewYdbConnection(config, credentialsProviderFactory)) , Config(config) , EntityIdGenerator(entityIdGenerator) { @@ -1181,8 +1181,8 @@ TExecDataQuerySettings NYq::TCheckpointStorage::DefaultExecDataQuerySettings() { //////////////////////////////////////////////////////////////////////////////// -TCheckpointStoragePtr NewYdbCheckpointStorage( - const NConfig::TYdbStorageConfig& config, +TCheckpointStoragePtr NewYdbCheckpointStorage( + const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const IEntityIdGenerator::TPtr& entityIdGenerator) { diff --git a/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h b/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h index 93d95f9be5f..c52b006f916 100644 --- a/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h +++ b/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h @@ -10,8 +10,8 @@ namespace NYq { //////////////////////////////////////////////////////////////////////////////// -TCheckpointStoragePtr NewYdbCheckpointStorage( - const NConfig::TYdbStorageConfig& config, +TCheckpointStoragePtr NewYdbCheckpointStorage( + const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const IEntityIdGenerator::TPtr& entityIdGenerator); diff --git a/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp b/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp index d7050a003be..ab209ec1e89 100644 --- a/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp +++ b/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp @@ -150,9 +150,9 @@ class TStateStorage : public IStateStorage { const NConfig::TYdbStorageConfig Config; public: - explicit TStateStorage( - const NConfig::TYdbStorageConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); + explicit TStateStorage( + const NConfig::TYdbStorageConfig& config, + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); ~TStateStorage() = default; TFuture<TIssues> Init() override; @@ -186,10 +186,10 @@ public: //////////////////////////////////////////////////////////////////////////////// -TStateStorage::TStateStorage( - const NConfig::TYdbStorageConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) - : YdbConnection(NewYdbConnection(config, credentialsProviderFactory)) +TStateStorage::TStateStorage( + const NConfig::TYdbStorageConfig& config, + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) + : YdbConnection(NewYdbConnection(config, credentialsProviderFactory)) , Config(config) { } @@ -554,11 +554,11 @@ TFuture<TStatus> TStateStorage::UpsertState(const TContextPtr& context) { //////////////////////////////////////////////////////////////////////////////// -TStateStoragePtr NewYdbStateStorage( - const NConfig::TYdbStorageConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) +TStateStoragePtr NewYdbStateStorage( + const NConfig::TYdbStorageConfig& config, + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) { - return new TStateStorage(config, credentialsProviderFactory); + return new TStateStorage(config, credentialsProviderFactory); } } // namespace NYq diff --git a/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h b/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h index 4faf4f17715..61dba973de7 100644 --- a/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h +++ b/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h @@ -9,8 +9,8 @@ namespace NYq { //////////////////////////////////////////////////////////////////////////////// -TStateStoragePtr NewYdbStateStorage( - const NConfig::TYdbStorageConfig& config, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); +TStateStoragePtr NewYdbStateStorage( + const NConfig::TYdbStorageConfig& config, + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); } // namespace NYq diff --git a/ydb/core/yq/libs/control_plane_storage/control_plane_storage.h b/ydb/core/yq/libs/control_plane_storage/control_plane_storage.h index ba38450e984..5180aedeea7 100644 --- a/ydb/core/yq/libs/control_plane_storage/control_plane_storage.h +++ b/ydb/core/yq/libs/control_plane_storage/control_plane_storage.h @@ -38,11 +38,11 @@ NActors::TActorId ControlPlaneStorageServiceActorId(ui32 nodeId = 0); NActors::IActor* CreateInMemoryControlPlaneStorageServiceActor(const NConfig::TControlPlaneStorageConfig& config); -NActors::IActor* CreateYdbControlPlaneStorageServiceActor( +NActors::IActor* CreateYdbControlPlaneStorageServiceActor( const NConfig::TControlPlaneStorageConfig& config, const NConfig::TCommonConfig& common, - const NMonitoring::TDynamicCounterPtr& counters, - const NYq::TYqSharedResources::TPtr& yqSharedResources, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); + const NMonitoring::TDynamicCounterPtr& counters, + const NYq::TYqSharedResources::TPtr& yqSharedResources, + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory); } // namespace NYq diff --git a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp index a91809bbe91..783b9242385 100644 --- a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp +++ b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp @@ -79,18 +79,18 @@ private: hFunc(TEvControlPlaneStorage::TEvDescribeBindingRequest, Handle); hFunc(TEvControlPlaneStorage::TEvModifyBindingRequest, Handle); hFunc(TEvControlPlaneStorage::TEvDeleteBindingRequest, Handle); - hFunc(TEvControlPlaneStorage::TEvDescribeJobRequest, Handle); - hFunc(TEvControlPlaneStorage::TEvWriteResultDataRequest, Handle); - hFunc(TEvControlPlaneStorage::TEvGetTaskRequest, Handle); - hFunc(TEvControlPlaneStorage::TEvPingTaskRequest, Handle); - hFunc(TEvControlPlaneStorage::TEvNodesHealthCheckRequest, Handle); - hFunc(NActors::NMon::TEvHttpInfo, Handle); + hFunc(TEvControlPlaneStorage::TEvDescribeJobRequest, Handle); + hFunc(TEvControlPlaneStorage::TEvWriteResultDataRequest, Handle); + hFunc(TEvControlPlaneStorage::TEvGetTaskRequest, Handle); + hFunc(TEvControlPlaneStorage::TEvPingTaskRequest, Handle); + hFunc(TEvControlPlaneStorage::TEvNodesHealthCheckRequest, Handle); + hFunc(NActors::NMon::TEvHttpInfo, Handle); ) void Handle(TEvControlPlaneStorage::TEvCreateQueryRequest::TPtr& ev) { - CPS_LOG_I("CreateQueryRequest"); - + CPS_LOG_I("CreateQueryRequest"); + const YandexQuery::CreateQueryRequest& request = ev->Get()->Request; CPS_LOG_D("CreateQueryRequest: " << request.DebugString()); CleanupIndempotencyKeys(); @@ -138,25 +138,25 @@ private: void Handle(TEvControlPlaneStorage::TEvListQueriesRequest::TPtr& ev) { - SendEmptyResponse< - TEvControlPlaneStorage::TEvListQueriesRequest::TPtr, - YandexQuery::ListQueriesResult, - TEvControlPlaneStorage::TEvListQueriesResponse>(ev, "ListQueriesRequest"); + SendEmptyResponse< + TEvControlPlaneStorage::TEvListQueriesRequest::TPtr, + YandexQuery::ListQueriesResult, + TEvControlPlaneStorage::TEvListQueriesResponse>(ev, "ListQueriesRequest"); } void Handle(TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr& ev) { - SendEmptyResponse< - TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr, - YandexQuery::DescribeQueryResult, - TEvControlPlaneStorage::TEvDescribeQueryResponse>(ev, "DescribeQueryRequest"); + SendEmptyResponse< + TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr, + YandexQuery::DescribeQueryResult, + TEvControlPlaneStorage::TEvDescribeQueryResponse>(ev, "DescribeQueryRequest"); } void Handle(TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr, - YandexQuery::ModifyQueryResult, + TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr, + YandexQuery::ModifyQueryResult, TEvControlPlaneStorage::TEvModifyQueryResponse, TAuditDetails<YandexQuery::Query>>(ev, "ModifyQueryRequest"); } @@ -164,8 +164,8 @@ private: void Handle(TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr, - YandexQuery::DeleteQueryResult, + TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr, + YandexQuery::DeleteQueryResult, TEvControlPlaneStorage::TEvDeleteQueryResponse, TAuditDetails<YandexQuery::Query>>(ev, "DeleteQueryRequest"); } @@ -173,58 +173,58 @@ private: void Handle(TEvControlPlaneStorage::TEvControlQueryRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvControlQueryRequest::TPtr, - YandexQuery::ControlQueryResult, + TEvControlPlaneStorage::TEvControlQueryRequest::TPtr, + YandexQuery::ControlQueryResult, TEvControlPlaneStorage::TEvControlQueryResponse, TAuditDetails<YandexQuery::Query>>(ev, "ControlQueryRequest"); } void Handle(TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr& ev) { - SendEmptyResponse< - TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr, - YandexQuery::GetResultDataResult, - TEvControlPlaneStorage::TEvGetResultDataResponse>(ev, "GetResultDataRequest"); + SendEmptyResponse< + TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr, + YandexQuery::GetResultDataResult, + TEvControlPlaneStorage::TEvGetResultDataResponse>(ev, "GetResultDataRequest"); } void Handle(TEvControlPlaneStorage::TEvListJobsRequest::TPtr& ev) { - SendEmptyResponse< - TEvControlPlaneStorage::TEvListJobsRequest::TPtr, - YandexQuery::ListJobsResult, - TEvControlPlaneStorage::TEvListJobsResponse>(ev, "ListJobsRequest"); + SendEmptyResponse< + TEvControlPlaneStorage::TEvListJobsRequest::TPtr, + YandexQuery::ListJobsResult, + TEvControlPlaneStorage::TEvListJobsResponse>(ev, "ListJobsRequest"); } void Handle(TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr, - YandexQuery::CreateConnectionResult, + TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr, + YandexQuery::CreateConnectionResult, TEvControlPlaneStorage::TEvCreateConnectionResponse, TAuditDetails<YandexQuery::Connection>>(ev, "CreateConnectionRequest"); } void Handle(TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr& ev) { - SendEmptyResponse< - TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr, - YandexQuery::ListConnectionsResult, - TEvControlPlaneStorage::TEvListConnectionsResponse>(ev, "ListConnectionsRequest"); + SendEmptyResponse< + TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr, + YandexQuery::ListConnectionsResult, + TEvControlPlaneStorage::TEvListConnectionsResponse>(ev, "ListConnectionsRequest"); } void Handle(TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr& ev) { - SendEmptyResponse< - TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr, - YandexQuery::DescribeConnectionResult, - TEvControlPlaneStorage::TEvDescribeConnectionResponse>(ev, "DescribeConnectionRequest"); + SendEmptyResponse< + TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr, + YandexQuery::DescribeConnectionResult, + TEvControlPlaneStorage::TEvDescribeConnectionResponse>(ev, "DescribeConnectionRequest"); } void Handle(TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr, - YandexQuery::ModifyConnectionResult, + TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr, + YandexQuery::ModifyConnectionResult, TEvControlPlaneStorage::TEvModifyConnectionResponse, TAuditDetails<YandexQuery::Connection>>(ev, "ModifyConnectionRequest"); } @@ -232,8 +232,8 @@ private: void Handle(TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr, - YandexQuery::DeleteConnectionResult, + TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr, + YandexQuery::DeleteConnectionResult, TEvControlPlaneStorage::TEvDeleteConnectionResponse, TAuditDetails<YandexQuery::Connection>>(ev, "DeleteConnectionRequest"); } @@ -241,33 +241,33 @@ private: void Handle(TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr, - YandexQuery::CreateBindingResult, + TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr, + YandexQuery::CreateBindingResult, TEvControlPlaneStorage::TEvCreateBindingResponse, TAuditDetails<YandexQuery::Binding>>(ev, "CreateBindingRequest"); } void Handle(TEvControlPlaneStorage::TEvListBindingsRequest::TPtr& ev) { - SendEmptyResponse< - TEvControlPlaneStorage::TEvListBindingsRequest::TPtr, - YandexQuery::ListBindingsResult, - TEvControlPlaneStorage::TEvListBindingsResponse>(ev, "ListBindingsRequest"); + SendEmptyResponse< + TEvControlPlaneStorage::TEvListBindingsRequest::TPtr, + YandexQuery::ListBindingsResult, + TEvControlPlaneStorage::TEvListBindingsResponse>(ev, "ListBindingsRequest"); } void Handle(TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr& ev) { - SendEmptyResponse< - TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr, - YandexQuery::DescribeBindingResult, - TEvControlPlaneStorage::TEvDescribeBindingResponse>(ev, "DescribeBindingRequest"); + SendEmptyResponse< + TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr, + YandexQuery::DescribeBindingResult, + TEvControlPlaneStorage::TEvDescribeBindingResponse>(ev, "DescribeBindingRequest"); } void Handle(TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr, - YandexQuery::ModifyBindingResult, + TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr, + YandexQuery::ModifyBindingResult, TEvControlPlaneStorage::TEvModifyBindingResponse, TAuditDetails<YandexQuery::Binding>>(ev, "ModifyBindingRequest"); } @@ -275,67 +275,67 @@ private: void Handle(TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr& ev) { SendEmptyAuditResponse< - TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr, - YandexQuery::DeleteBindingResult, + TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr, + YandexQuery::DeleteBindingResult, TEvControlPlaneStorage::TEvDeleteBindingResponse, TAuditDetails<YandexQuery::Binding>>(ev, "DeleteBindingRequest"); } - void Handle(TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr, - YandexQuery::DescribeJobResult, - TEvControlPlaneStorage::TEvDescribeJobResponse>(ev, "DescribeJobRequest"); - } - - void Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr, - NYql::TIssues, - TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev, "WriteResultDataRequest"); - } - - void Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev) - { - CPS_LOG_I("GetTaskRequest"); - TVector<TEvControlPlaneStorage::TTask> tasks; - TString owner; - auto event = std::make_unique<TEvControlPlaneStorage::TEvGetTaskResponse>(tasks, owner); - NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie)); - } - - void Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvPingTaskRequest::TPtr, - YandexQuery::QueryAction, - TEvControlPlaneStorage::TEvPingTaskResponse>(ev, "PingTaskRequest"); - } - - void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev) - { - SendEmptyResponse< - TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr, + void Handle(TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr& ev) + { + SendEmptyResponse< + TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr, + YandexQuery::DescribeJobResult, + TEvControlPlaneStorage::TEvDescribeJobResponse>(ev, "DescribeJobRequest"); + } + + void Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev) + { + SendEmptyResponse< + TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr, + NYql::TIssues, + TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev, "WriteResultDataRequest"); + } + + void Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev) + { + CPS_LOG_I("GetTaskRequest"); + TVector<TEvControlPlaneStorage::TTask> tasks; + TString owner; + auto event = std::make_unique<TEvControlPlaneStorage::TEvGetTaskResponse>(tasks, owner); + NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie)); + } + + void Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev) + { + SendEmptyResponse< + TEvControlPlaneStorage::TEvPingTaskRequest::TPtr, + YandexQuery::QueryAction, + TEvControlPlaneStorage::TEvPingTaskResponse>(ev, "PingTaskRequest"); + } + + void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev) + { + SendEmptyResponse< + TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr, Yq::Private::NodesHealthCheckResult, - TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev, "NodesHealthCheckRequest"); - } - - void Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) { - TStringStream str; - Send(ev->Sender, new NActors::NMon::TEvHttpInfoRes(str.Str())); - } - - template<typename TRequest, typename TResult, typename TEvResult> - void SendEmptyResponse(TRequest& ev, std::string logText) { - CPS_LOG_I(logText); - + TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev, "NodesHealthCheckRequest"); + } + + void Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) { + TStringStream str; + Send(ev->Sender, new NActors::NMon::TEvHttpInfoRes(str.Str())); + } + + template<typename TRequest, typename TResult, typename TEvResult> + void SendEmptyResponse(TRequest& ev, std::string logText) { + CPS_LOG_I(logText); + TResult result = {}; - auto event = std::make_unique<TEvResult>(result); - NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie)); - } - + auto event = std::make_unique<TEvResult>(result); + NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie)); + } + template<typename TRequest, typename TResult, typename TEvResult, typename TAuditDetails> void SendEmptyAuditResponse(TRequest& ev, std::string logText) { CPS_LOG_I(logText); diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp index b41fa5f37b9..222ebb6e8e3 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp @@ -3,7 +3,7 @@ #include "ydb_control_plane_storage_impl.h" #include <ydb/library/security/ydb_credentials_provider_factory.h> - + namespace NYq { namespace { @@ -670,12 +670,12 @@ TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite( return promise.GetFuture(); } -NActors::IActor* CreateYdbControlPlaneStorageServiceActor( +NActors::IActor* CreateYdbControlPlaneStorageServiceActor( const NConfig::TControlPlaneStorageConfig& config, const NConfig::TCommonConfig& common, - const NMonitoring::TDynamicCounterPtr& counters, - const ::NYq::TYqSharedResources::TPtr& yqSharedResources, - const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) { + const NMonitoring::TDynamicCounterPtr& counters, + const ::NYq::TYqSharedResources::TPtr& yqSharedResources, + const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) { return new TYdbControlPlaneStorageActor(config, common, counters, yqSharedResources, credentialsProviderFactory); } diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h index 34f9fc5c710..1c8cda7ee72 100644 --- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h +++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h @@ -183,20 +183,20 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped<TYdbCont static constexpr int64_t InitialRevision = 1; - NKikimr::TYdbCredentialsProviderFactory CredProviderFactory; - + NKikimr::TYdbCredentialsProviderFactory CredProviderFactory; + public: - TYdbControlPlaneStorageActor( + TYdbControlPlaneStorageActor( const NConfig::TControlPlaneStorageConfig& config, const NConfig::TCommonConfig& common, - const NMonitoring::TDynamicCounterPtr& counters, - const ::NYq::TYqSharedResources::TPtr& yqSharedResources, - const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory) + const NMonitoring::TDynamicCounterPtr& counters, + const ::NYq::TYqSharedResources::TPtr& yqSharedResources, + const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory) : Counters(counters) , FinalStatusCounters(counters) , Config(config, common) , YqSharedResources(yqSharedResources) - , CredProviderFactory(credProviderFactory) + , CredProviderFactory(credProviderFactory) { } diff --git a/ydb/core/yq/libs/events/event_subspace.h b/ydb/core/yq/libs/events/event_subspace.h index 9eb8b2f3a66..99814caaae0 100644 --- a/ydb/core/yq/libs/events/event_subspace.h +++ b/ydb/core/yq/libs/events/event_subspace.h @@ -24,7 +24,7 @@ struct TYqEventSubspace { ConfigUpdater, ControlPlaneStorage, ControlPlaneProxy, - AuditService, + AuditService, TestConnection, SubspacesEnd, diff --git a/ydb/core/yq/libs/init/init.cpp b/ydb/core/yq/libs/init/init.cpp index fc4ad80f00d..d66ac202bfa 100644 --- a/ydb/core/yq/libs/init/init.cpp +++ b/ydb/core/yq/libs/init/init.cpp @@ -103,7 +103,7 @@ void Init( const TString& tenant, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const IYqSharedResources::TPtr& iyqSharedResources, - const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory, + const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory, const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const ui32& icPort @@ -124,7 +124,7 @@ void Init( credentialsProviderFactory); actorRegistrator(NYq::ControlPlaneStorageServiceActorId(), controlPlaneStorage); } - + if (protoConfig.GetTestConnection().GetEnabled()) { auto testConnection = NYq::CreateTestConnectionActor( protoConfig.GetTestConnection(), @@ -270,11 +270,11 @@ void Init( } } -IYqSharedResources::TPtr CreateYqSharedResources( +IYqSharedResources::TPtr CreateYqSharedResources( const NYq::NConfig::TConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters) -{ +{ return CreateYqSharedResourcesImpl(config, credentialsProviderFactory, counters); } diff --git a/ydb/core/yq/libs/init/init.h b/ydb/core/yq/libs/init/init.h index 1487fc950bd..fa26651f185 100644 --- a/ydb/core/yq/libs/init/init.h +++ b/ydb/core/yq/libs/init/init.h @@ -21,7 +21,7 @@ namespace NYq { using TActorRegistrator = std::function<void(NActors::TActorId, NActors::IActor*)>; -IYqSharedResources::TPtr CreateYqSharedResources( +IYqSharedResources::TPtr CreateYqSharedResources( const NYq::NConfig::TConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters); @@ -34,7 +34,7 @@ void Init( const TString& tenant, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const IYqSharedResources::TPtr& yqSharedResources, - const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory, + const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory, const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const ui32& icPort diff --git a/ydb/core/yq/libs/shared_resources/db_pool.cpp b/ydb/core/yq/libs/shared_resources/db_pool.cpp index 125802a0d01..372dfd72cdf 100644 --- a/ydb/core/yq/libs/shared_resources/db_pool.cpp +++ b/ydb/core/yq/libs/shared_resources/db_pool.cpp @@ -221,22 +221,22 @@ static void PrepareConfig(NYq::NConfig::TDbPoolConfig& config) { } } -TDbPoolMap::TDbPoolMap( +TDbPoolMap::TDbPoolMap( const NYq::NConfig::TDbPoolConfig& config, - NYdb::TDriver driver, + NYdb::TDriver driver, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters) : Config(config) , Driver(driver) - , CredentialsProviderFactory(credentialsProviderFactory) + , CredentialsProviderFactory(credentialsProviderFactory) , Counters(counters) { PrepareConfig(Config); } -TDbPoolHolder::TDbPoolHolder( +TDbPoolHolder::TDbPoolHolder( const NYq::NConfig::TDbPoolConfig& config, - const NYdb::TDriver& driver, + const NYdb::TDriver& driver, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters) : Driver(driver) @@ -286,12 +286,12 @@ TDbPool::TPtr TDbPoolMap::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) { .Database(Config.GetStorage().GetDatabase()) .DiscoveryEndpoint(Config.GetStorage().GetEndpoint()) .DiscoveryMode(NYdb::EDiscoveryMode::Async); - NKikimr::TYdbCredentialsSettings credSettings; + NKikimr::TYdbCredentialsSettings credSettings; credSettings.UseLocalMetadata = Config.GetStorage().GetUseLocalMetadataService(); credSettings.OAuthToken = Config.GetStorage().GetToken(); - - clientSettings.CredentialsProviderFactory(CredentialsProviderFactory(credSettings)); - + + clientSettings.CredentialsProviderFactory(CredentialsProviderFactory(credSettings)); + clientSettings.EnableSsl(Config.GetStorage().GetUseSsl()); TableClient = MakeHolder<NYdb::NTable::TTableClient>(Driver, clientSettings); diff --git a/ydb/core/yq/libs/shared_resources/db_pool.h b/ydb/core/yq/libs/shared_resources/db_pool.h index 6e6dec32e64..68d9adc2fca 100644 --- a/ydb/core/yq/libs/shared_resources/db_pool.h +++ b/ydb/core/yq/libs/shared_resources/db_pool.h @@ -5,7 +5,7 @@ #include <ydb/core/yq/libs/events/events.h> #include <ydb/library/security/ydb_credentials_provider_factory.h> - + #include <library/cpp/actors/core/actor.h> #include <library/cpp/monlib/dynamic_counters/counters.h> @@ -57,16 +57,16 @@ private: NYdb::TDriver Driver; THashMap<EDbPoolId, TDbPool::TPtr> Pools; THolder<NYdb::NTable::TTableClient> TableClient; - NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory; + NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory; const NMonitoring::TDynamicCounterPtr Counters; }; class TDbPoolHolder: public TThrRefBase { public: using TPtr = TIntrusivePtr<TDbPoolHolder>; - TDbPoolHolder( + TDbPoolHolder( const NYq::NConfig::TDbPoolConfig& config, - const NYdb::TDriver& driver, + const NYdb::TDriver& driver, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters); diff --git a/ydb/core/yq/libs/shared_resources/shared_resources.cpp b/ydb/core/yq/libs/shared_resources/shared_resources.cpp index f84c3dfc4cd..3dfd566232f 100644 --- a/ydb/core/yq/libs/shared_resources/shared_resources.cpp +++ b/ydb/core/yq/libs/shared_resources/shared_resources.cpp @@ -84,7 +84,7 @@ struct TActorSystemPtrMixin { }; struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedResources { - explicit TYqSharedResourcesImpl( + explicit TYqSharedResourcesImpl( const NYq::NConfig::TConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters) @@ -118,7 +118,7 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes return cfg; } - void CreateDbPoolHolder( + void CreateDbPoolHolder( const NYq::NConfig::TDbPoolConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters) { @@ -128,7 +128,7 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes } // namespace -TYqSharedResources::TPtr CreateYqSharedResourcesImpl( +TYqSharedResources::TPtr CreateYqSharedResourcesImpl( const NYq::NConfig::TConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters) { diff --git a/ydb/core/yq/libs/shared_resources/shared_resources.h b/ydb/core/yq/libs/shared_resources/shared_resources.h index 3f047e653af..bfd1ec66953 100644 --- a/ydb/core/yq/libs/shared_resources/shared_resources.h +++ b/ydb/core/yq/libs/shared_resources/shared_resources.h @@ -6,7 +6,7 @@ #include <ydb/core/yq/libs/shared_resources/interface/shared_resources.h> #include <ydb/library/security/ydb_credentials_provider_factory.h> - + #include <library/cpp/actors/core/actorsystem.h> namespace NYq { @@ -23,7 +23,7 @@ protected: explicit TYqSharedResources(NYdb::TDriver driver); }; -TYqSharedResources::TPtr CreateYqSharedResourcesImpl( +TYqSharedResources::TPtr CreateYqSharedResourcesImpl( const NYq::NConfig::TConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters); diff --git a/ydb/core/yq/libs/ya.make b/ydb/core/yq/libs/ya.make index 914fee971d1..ccc88a55b62 100644 --- a/ydb/core/yq/libs/ya.make +++ b/ydb/core/yq/libs/ya.make @@ -2,7 +2,7 @@ OWNER(g:yq) RECURSE( actors - audit + audit checkpoint_storage checkpointing checkpointing_common diff --git a/ydb/core/yq/libs/ydb/ydb.cpp b/ydb/core/yq/libs/ydb/ydb.cpp index 4230c93d00c..d6096abce32 100644 --- a/ydb/core/yq/libs/ydb/ydb.cpp +++ b/ydb/core/yq/libs/ydb/ydb.cpp @@ -205,14 +205,14 @@ NYdb::TDriverConfig GetDriverConfig(const NConfig::TYdbStorageConfig& config, .SetEndpoint(config.GetEndpoint()) .SetDatabase(config.GetDatabase()); - NKikimr::TYdbCredentialsSettings credSettings; - credSettings.UseLocalMetadata = config.GetUseLocalMetadataService(); - credSettings.OAuthToken = oauth; - credSettings.SaKeyFile = config.GetSaKeyFile(); - credSettings.IamEndpoint = config.GetIamEndpoint(); - - driverConfig.SetCredentialsProviderFactory(credProviderFactory(credSettings)); - + NKikimr::TYdbCredentialsSettings credSettings; + credSettings.UseLocalMetadata = config.GetUseLocalMetadataService(); + credSettings.OAuthToken = oauth; + credSettings.SaKeyFile = config.GetSaKeyFile(); + credSettings.IamEndpoint = config.GetIamEndpoint(); + + driverConfig.SetCredentialsProviderFactory(credProviderFactory(credSettings)); + if (config.GetUseLocalMetadataService()) { driverConfig.UseSecureConnection(); } @@ -227,7 +227,7 @@ NYdb::TDriverConfig GetDriverConfig(const NConfig::TYdbStorageConfig& config, TYdbConnectionPtr NewYdbConnection(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory) { - auto driverConfig = GetDriverConfig(config, credProviderFactory); + auto driverConfig = GetDriverConfig(config, credProviderFactory); return MakeIntrusive<TYdbConnection>(driverConfig, config); } diff --git a/ydb/core/yq/libs/ydb/ydb.h b/ydb/core/yq/libs/ydb/ydb.h index abe8d179090..14c07422883 100644 --- a/ydb/core/yq/libs/ydb/ydb.h +++ b/ydb/core/yq/libs/ydb/ydb.h @@ -94,9 +94,9 @@ using TGenerationContextPtr = TIntrusivePtr<TGenerationContext>; //////////////////////////////////////////////////////////////////////////////// -NYdb::TDriverConfig GetDriverConfig(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory); +NYdb::TDriverConfig GetDriverConfig(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory); -TYdbConnectionPtr NewYdbConnection(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory); +TYdbConnectionPtr NewYdbConnection(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory); NYdb::TStatus MakeErrorStatus( NYdb::EStatus code, diff --git a/ydb/library/security/ya.make b/ydb/library/security/ya.make index 3fd5c726d0e..ce048987e4a 100644 --- a/ydb/library/security/ya.make +++ b/ydb/library/security/ya.make @@ -12,7 +12,7 @@ PEERDIR( SRCS( util.cpp - ydb_credentials_provider_factory.cpp + ydb_credentials_provider_factory.cpp ) END() diff --git a/ydb/library/security/ydb_credentials_provider_factory.cpp b/ydb/library/security/ydb_credentials_provider_factory.cpp index 010f16d3f27..25174da6f05 100644 --- a/ydb/library/security/ydb_credentials_provider_factory.cpp +++ b/ydb/library/security/ydb_credentials_provider_factory.cpp @@ -1,13 +1,13 @@ -#include "ydb_credentials_provider_factory.h" - - -namespace NKikimr { - -std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateYdbCredentialsProviderFactory(const TYdbCredentialsSettings& settings) -{ - return settings.OAuthToken - ? NYdb::CreateOAuthCredentialsProviderFactory(settings.OAuthToken) - : NYdb::CreateInsecureCredentialsProviderFactory(); -} - -} +#include "ydb_credentials_provider_factory.h" + + +namespace NKikimr { + +std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateYdbCredentialsProviderFactory(const TYdbCredentialsSettings& settings) +{ + return settings.OAuthToken + ? NYdb::CreateOAuthCredentialsProviderFactory(settings.OAuthToken) + : NYdb::CreateInsecureCredentialsProviderFactory(); +} + +} diff --git a/ydb/library/security/ydb_credentials_provider_factory.h b/ydb/library/security/ydb_credentials_provider_factory.h index 3c0779de87f..92c986e2490 100644 --- a/ydb/library/security/ydb_credentials_provider_factory.h +++ b/ydb/library/security/ydb_credentials_provider_factory.h @@ -1,21 +1,21 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> - -#include <memory> - -namespace NKikimr { - -struct TYdbCredentialsSettings { - bool UseLocalMetadata = false; - TString OAuthToken; - - TString SaKeyFile; - TString IamEndpoint; -}; - -using TYdbCredentialsProviderFactory = std::function<std::shared_ptr<NYdb::ICredentialsProviderFactory>(const TYdbCredentialsSettings& settings)>; - -std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateYdbCredentialsProviderFactory(const TYdbCredentialsSettings& settings); - -} + +#include <memory> + +namespace NKikimr { + +struct TYdbCredentialsSettings { + bool UseLocalMetadata = false; + TString OAuthToken; + + TString SaKeyFile; + TString IamEndpoint; +}; + +using TYdbCredentialsProviderFactory = std::function<std::shared_ptr<NYdb::ICredentialsProviderFactory>(const TYdbCredentialsSettings& settings)>; + +std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateYdbCredentialsProviderFactory(const TYdbCredentialsSettings& settings); + +} diff --git a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp index 143ecaab35a..215a1e9f692 100644 --- a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp @@ -731,11 +731,11 @@ TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, cons auto traits = TCoHoppingTraits(hoppingSetting->Child(1)); auto timeExtractor = traits.TimeExtractor(); - auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); - for (const auto& usedField : usedType->GetItems()) { - usedFields.insert(usedField->GetName()); - } - + auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); + for (const auto& usedField : usedType->GetItems()) { + usedFields.insert(usedField->GetName()); + } + TSet<TStringBuf> lambdaSubset; if (HaveFieldsSubset(timeExtractor.Body().Ptr(), *timeExtractor.Args().Arg(0).Raw(), lambdaSubset, parentsMap)) { usedFields.insert(lambdaSubset.cbegin(), lambdaSubset.cend()); diff --git a/ydb/library/yql/core/common_opt/yql_co_flow2.cpp b/ydb/library/yql/core/common_opt/yql_co_flow2.cpp index 754f4990a6c..88f91400319 100644 --- a/ydb/library/yql/core/common_opt/yql_co_flow2.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_flow2.cpp @@ -67,11 +67,11 @@ TExprNode::TPtr AggregateSubsetFieldsAnalyzer(const TCoAggregate& node, TExprCon auto traits = TCoHoppingTraits(hoppingSetting->Child(1)); auto timeExtractor = traits.TimeExtractor(); - auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); - for (const auto& usedField : usedType->GetItems()) { - usedFields.insert(usedField->GetName()); - } - + auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); + for (const auto& usedField : usedType->GetItems()) { + usedFields.insert(usedField->GetName()); + } + TSet<TStringBuf> lambdaSubset; if (!HaveFieldsSubset(timeExtractor.Body().Ptr(), *timeExtractor.Args().Arg(0).Raw(), lambdaSubset, parentsMap)) { return node.Ptr(); @@ -2019,42 +2019,42 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) { .Build(); }; - map[TCoHoppingTraits::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { - TCoHoppingTraits self(node); - - auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); - - const auto lambdaBody = self.TimeExtractor().Body().Ptr(); - const auto& arg = self.TimeExtractor().Args().Arg(0).Ref(); - - TSet<TStringBuf> usedFields; - if (!HaveFieldsSubset(lambdaBody, arg, usedFields, *optCtx.ParentsMap)) { - return node; - } - - if (usedFields.size() == structType->GetSize()) { - return node; - } - - TVector<const TItemExprType*> subsetItems; - for (const auto& item : structType->GetItems()) { - if (usedFields.contains(item->GetName())) { - subsetItems.push_back(item); - } - } - - auto subsetType = ctx.MakeType<TStructExprType>(subsetItems); - YQL_CLOG(DEBUG, Core) << "FieldSubset for HoppingTraits"; - return Build<TCoHoppingTraits>(ctx, node->Pos()) - .ItemType(ExpandType(node->Pos(), *subsetType, ctx)) - .TimeExtractor(ctx.DeepCopyLambda(self.TimeExtractor().Ref())) - .Hop(self.Hop()) - .Interval(self.Interval()) - .Delay(self.Delay()) - .DataWatermarks(self.DataWatermarks()) - .Done().Ptr(); - }; - + map[TCoHoppingTraits::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { + TCoHoppingTraits self(node); + + auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>(); + + const auto lambdaBody = self.TimeExtractor().Body().Ptr(); + const auto& arg = self.TimeExtractor().Args().Arg(0).Ref(); + + TSet<TStringBuf> usedFields; + if (!HaveFieldsSubset(lambdaBody, arg, usedFields, *optCtx.ParentsMap)) { + return node; + } + + if (usedFields.size() == structType->GetSize()) { + return node; + } + + TVector<const TItemExprType*> subsetItems; + for (const auto& item : structType->GetItems()) { + if (usedFields.contains(item->GetName())) { + subsetItems.push_back(item); + } + } + + auto subsetType = ctx.MakeType<TStructExprType>(subsetItems); + YQL_CLOG(DEBUG, Core) << "FieldSubset for HoppingTraits"; + return Build<TCoHoppingTraits>(ctx, node->Pos()) + .ItemType(ExpandType(node->Pos(), *subsetType, ctx)) + .TimeExtractor(ctx.DeepCopyLambda(self.TimeExtractor().Ref())) + .Hop(self.Hop()) + .Interval(self.Interval()) + .Delay(self.Delay()) + .DataWatermarks(self.DataWatermarks()) + .Done().Ptr(); + }; + map["AggregationTraits"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) { auto type = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType(); if (type->GetKind() != ETypeAnnotationKind::Struct) { diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json index b667a4f74f1..edfcaa15302 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json @@ -552,12 +552,12 @@ "Base": "TCallable", "Match": {"Type": "Callable", "Name": "HoppingTraits"}, "Children": [ - {"Index": 0, "Name": "ItemType", "Type": "TExprBase"}, - {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"}, - {"Index": 2, "Name": "Hop", "Type": "TExprBase"}, - {"Index": 3, "Name": "Interval", "Type": "TExprBase"}, - {"Index": 4, "Name": "Delay", "Type": "TExprBase"}, - {"Index": 5, "Name": "DataWatermarks", "Type": "TExprBase"} + {"Index": 0, "Name": "ItemType", "Type": "TExprBase"}, + {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"}, + {"Index": 2, "Name": "Hop", "Type": "TExprBase"}, + {"Index": 3, "Name": "Interval", "Type": "TExprBase"}, + {"Index": 4, "Name": "Delay", "Type": "TExprBase"}, + {"Index": 5, "Name": "DataWatermarks", "Type": "TExprBase"} ] }, { @@ -1601,38 +1601,38 @@ "Base": "TCoInputBase", "Match": {"Type": "Callable", "Name": "HoppingCore"}, "Children": [ - {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"}, - {"Index": 2, "Name": "Hop", "Type": "TExprBase"}, - {"Index": 3, "Name": "Interval", "Type": "TExprBase"}, - {"Index": 4, "Name": "Delay", "Type": "TExprBase"}, - {"Index": 5, "Name": "InitHandler", "Type": "TCoLambda"}, - {"Index": 6, "Name": "UpdateHandler", "Type": "TCoLambda"}, - {"Index": 7, "Name": "SaveHandler", "Type": "TExprBase"}, - {"Index": 8, "Name": "LoadHandler", "Type": "TExprBase"}, - {"Index": 9, "Name": "MergeHandler", "Type": "TCoLambda"}, - {"Index": 10, "Name": "FinishHandler", "Type": "TCoLambda"} - ] - }, - { - "Name": "TCoMultiHoppingCore", - "Base": "TCoInputBase", - "Match": {"Type": "Callable", "Name": "MultiHoppingCore"}, - "Children": [ - {"Index": 1, "Name": "KeyExtractor", "Type": "TCoLambda"}, - {"Index": 2, "Name": "TimeExtractor", "Type": "TCoLambda"}, - {"Index": 3, "Name": "Hop", "Type": "TExprBase"}, - {"Index": 4, "Name": "Interval", "Type": "TExprBase"}, - {"Index": 5, "Name": "Delay", "Type": "TExprBase"}, - {"Index": 6, "Name": "DataWatermarks", "Type": "TExprBase"}, - {"Index": 7, "Name": "InitHandler", "Type": "TCoLambda"}, - {"Index": 8, "Name": "UpdateHandler", "Type": "TCoLambda"}, - {"Index": 9, "Name": "SaveHandler", "Type": "TExprBase"}, - {"Index": 10, "Name": "LoadHandler", "Type": "TExprBase"}, - {"Index": 11, "Name": "MergeHandler", "Type": "TCoLambda"}, - {"Index": 12, "Name": "FinishHandler", "Type": "TCoLambda"} - ] - }, - { + {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"}, + {"Index": 2, "Name": "Hop", "Type": "TExprBase"}, + {"Index": 3, "Name": "Interval", "Type": "TExprBase"}, + {"Index": 4, "Name": "Delay", "Type": "TExprBase"}, + {"Index": 5, "Name": "InitHandler", "Type": "TCoLambda"}, + {"Index": 6, "Name": "UpdateHandler", "Type": "TCoLambda"}, + {"Index": 7, "Name": "SaveHandler", "Type": "TExprBase"}, + {"Index": 8, "Name": "LoadHandler", "Type": "TExprBase"}, + {"Index": 9, "Name": "MergeHandler", "Type": "TCoLambda"}, + {"Index": 10, "Name": "FinishHandler", "Type": "TCoLambda"} + ] + }, + { + "Name": "TCoMultiHoppingCore", + "Base": "TCoInputBase", + "Match": {"Type": "Callable", "Name": "MultiHoppingCore"}, + "Children": [ + {"Index": 1, "Name": "KeyExtractor", "Type": "TCoLambda"}, + {"Index": 2, "Name": "TimeExtractor", "Type": "TCoLambda"}, + {"Index": 3, "Name": "Hop", "Type": "TExprBase"}, + {"Index": 4, "Name": "Interval", "Type": "TExprBase"}, + {"Index": 5, "Name": "Delay", "Type": "TExprBase"}, + {"Index": 6, "Name": "DataWatermarks", "Type": "TExprBase"}, + {"Index": 7, "Name": "InitHandler", "Type": "TCoLambda"}, + {"Index": 8, "Name": "UpdateHandler", "Type": "TCoLambda"}, + {"Index": 9, "Name": "SaveHandler", "Type": "TExprBase"}, + {"Index": 10, "Name": "LoadHandler", "Type": "TExprBase"}, + {"Index": 11, "Name": "MergeHandler", "Type": "TCoLambda"}, + {"Index": 12, "Name": "FinishHandler", "Type": "TCoLambda"} + ] + }, + { "Name": "TCoCombineCore", "Base": "TCoInputBase", "Match": {"Type": "Callable", "Name": "CombineCore"}, diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index 269fccd5af3..5846e6cb108 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -11545,15 +11545,15 @@ template <NKikimr::NUdf::EDataSlot DataSlot> break; } } - for (auto& x : ctx.Types.DataSinks) { - auto tokens = x->GetClusterTokens(); - auto token = tokens ? tokens->FindPtr(clusterName) : nullptr; - if (token) { - clusterCred.ConstructInPlace(TString(x->GetName()), "", *token); - cred = clusterCred.Get(); - break; - } - } + for (auto& x : ctx.Types.DataSinks) { + auto tokens = x->GetClusterTokens(); + auto token = tokens ? tokens->FindPtr(clusterName) : nullptr; + if (token) { + clusterCred.ConstructInPlace(TString(x->GetName()), "", *token); + cred = clusterCred.Get(); + break; + } + } } } } @@ -12996,7 +12996,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["GroupingCore"] = &GroupingCoreWrapper; Functions["HoppingTraits"] = &HoppingTraitsWrapper; Functions["HoppingCore"] = &HoppingCoreWrapper; - Functions["MultiHoppingCore"] = &MultiHoppingCoreWrapper; + Functions["MultiHoppingCore"] = &MultiHoppingCoreWrapper; Functions["EquiJoin"] = &EquiJoinWrapper; Functions["OptionalReduce"] = &OptionalReduceWrapper; Functions["OptionalItemType"] = &TypeArgWrapper<ETypeArgument::OptionalItem>; diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp index ac56e2c71a8..b145cde3893 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp @@ -4546,7 +4546,7 @@ namespace { const auto item = tupleType->GetItems()[index]; rowColumns.push_back(ctx.Expr.MakeType<TItemExprType>( child->Head().Child(index)->Content(), - (isOptional || (input->Child(1)->ChildrenSize() == 0 && !isHopping)) && + (isOptional || (input->Child(1)->ChildrenSize() == 0 && !isHopping)) && item->GetKind() != ETypeAnnotationKind::Optional ? ctx.Expr.MakeType<TOptionalExprType>(item) : item)); } } else { @@ -5399,7 +5399,7 @@ namespace { IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); - if (!EnsureMinArgsCount(*input, 5, ctx.Expr) || !EnsureMaxArgsCount(*input, 6, ctx.Expr)) { + if (!EnsureMinArgsCount(*input, 5, ctx.Expr) || !EnsureMaxArgsCount(*input, 6, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) { @@ -5470,46 +5470,46 @@ namespace { return convertStatus; } - if (input->ChildrenSize() == 6) { - const auto& dataWatermarksNodePtr = input->ChildRef(5); - if (dataWatermarksNodePtr->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Unit) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(dataWatermarksNodePtr->Pos()), TStringBuilder() - << "Expected unit type, but got: " - << *dataWatermarksNodePtr->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - } - + if (input->ChildrenSize() == 6) { + const auto& dataWatermarksNodePtr = input->ChildRef(5); + if (dataWatermarksNodePtr->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Unit) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(dataWatermarksNodePtr->Pos()), TStringBuilder() + << "Expected unit type, but got: " + << *dataWatermarksNodePtr->GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + } + input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>()); return IGraphTransformer::TStatus::Ok; } IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); - if (!EnsureArgsCount(*input, 11, ctx.Expr)) { + if (!EnsureArgsCount(*input, 11, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - + auto& item = input->ChildRef(0); - auto& lambdaTimeExtractor = input->ChildRef(1); - auto* hop = input->Child(2); - auto* interval = input->Child(3); - auto* delay = input->Child(4); - - auto& lambdaInit = input->ChildRef(5); - auto& lambdaUpdate = input->ChildRef(6); - auto& saveLambda = input->ChildRef(7); - auto& loadLambda = input->ChildRef(8); - auto& lambdaMerge = input->ChildRef(9); - auto& lambdaFinish = input->ChildRef(10); - + auto& lambdaTimeExtractor = input->ChildRef(1); + auto* hop = input->Child(2); + auto* interval = input->Child(3); + auto* delay = input->Child(4); + + auto& lambdaInit = input->ChildRef(5); + auto& lambdaUpdate = input->ChildRef(6); + auto& saveLambda = input->ChildRef(7); + auto& loadLambda = input->ChildRef(8); + auto& lambdaMerge = input->ChildRef(9); + auto& lambdaFinish = input->ChildRef(10); + if (!EnsureStreamType(*item, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } - auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1)); + auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1)); status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2)); status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2)); status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 2)); @@ -5524,24 +5524,24 @@ namespace { return IGraphTransformer::TStatus::Error; } - if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) - { - return IGraphTransformer::TStatus::Error; - } - if (!lambdaTimeExtractor->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!EnsureSpecificDataType(*hop, EDataSlot::Interval, ctx.Expr, true)) { - return IGraphTransformer::TStatus::Error; - } - if (!EnsureSpecificDataType(*interval, EDataSlot::Interval, ctx.Expr, true)) { - return IGraphTransformer::TStatus::Error; - } - if (!EnsureSpecificDataType(*delay, EDataSlot::Interval, ctx.Expr, true)) { - return IGraphTransformer::TStatus::Error; - } - + if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) + { + return IGraphTransformer::TStatus::Error; + } + if (!lambdaTimeExtractor->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!EnsureSpecificDataType(*hop, EDataSlot::Interval, ctx.Expr, true)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureSpecificDataType(*interval, EDataSlot::Interval, ctx.Expr, true)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureSpecificDataType(*delay, EDataSlot::Interval, ctx.Expr, true)) { + return IGraphTransformer::TStatus::Error; + } + if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -5635,167 +5635,167 @@ namespace { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { - Y_UNUSED(output); - if (!EnsureArgsCount(*input, 13, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - auto& item = input->ChildRef(0); - auto& lambdaKeyExtractor = input->ChildRef(1); - - auto& lambdaTimeExtractor = input->ChildRef(2); - auto* hop = input->Child(3); - auto* interval = input->Child(4); - auto* delay = input->Child(5); - - auto& lambdaInit = input->ChildRef(7); - auto& lambdaUpdate = input->ChildRef(8); - auto& saveLambda = input->ChildRef(9); - auto& loadLambda = input->ChildRef(10); - auto& lambdaMerge = input->ChildRef(11); - auto& lambdaFinish = input->ChildRef(12); - - if (!EnsureStreamType(*item, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - auto status = ConvertToLambda(lambdaKeyExtractor, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1)); - status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1)); - status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2)); - status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2)); - status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 3)); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - auto timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp)); - auto itemType = item->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType(); - - if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaKeyExtractor, {itemType}, ctx.Expr)) - { - return IGraphTransformer::TStatus::Error; - } - auto keyType = lambdaKeyExtractor->GetTypeAnn(); - if (!keyType) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) - { - return IGraphTransformer::TStatus::Error; - } - if (!lambdaTimeExtractor->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!EnsureSpecificDataType(*hop, EDataSlot::Interval, ctx.Expr, true)) { - return IGraphTransformer::TStatus::Error; - } - if (!EnsureSpecificDataType(*interval, EDataSlot::Interval, ctx.Expr, true)) { - return IGraphTransformer::TStatus::Error; - } - if (!EnsureSpecificDataType(*delay, EDataSlot::Interval, ctx.Expr, true)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (!lambdaInit->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - auto stateType = lambdaInit->GetTypeAnn(); - if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (!lambdaUpdate->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " - << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (!lambdaMerge->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: " - << *stateType << ", but got: " << *lambdaMerge->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {keyType, stateType, timeType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (!lambdaFinish->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() << - "Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn())); - return IGraphTransformer::TStatus::Error; - } - - if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << - "Save and load lambdas must be specified at the same time")); - return IGraphTransformer::TStatus::Error; - } - - if (!saveLambda->IsCallable("Void")) { - auto status = ConvertToLambda(saveLambda, ctx.Expr, 1); - status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1)); - if (status.Level != IGraphTransformer::TStatus::Ok) { - return status; - } - - if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (!saveLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - auto savedStateType = saveLambda->GetTypeAnn(); - if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - - if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) { - return IGraphTransformer::TStatus::Error; - } - if (!loadLambda->GetTypeAnn()) { - return IGraphTransformer::TStatus::Repeat; - } - - if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " - << *loadLambda->GetTypeAnn() << " != " << *stateType)); - return IGraphTransformer::TStatus::Error; - } - } - - input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(lambdaFinish->GetTypeAnn())); - return IGraphTransformer::TStatus::Ok; - } - + IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 13, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + auto& item = input->ChildRef(0); + auto& lambdaKeyExtractor = input->ChildRef(1); + + auto& lambdaTimeExtractor = input->ChildRef(2); + auto* hop = input->Child(3); + auto* interval = input->Child(4); + auto* delay = input->Child(5); + + auto& lambdaInit = input->ChildRef(7); + auto& lambdaUpdate = input->ChildRef(8); + auto& saveLambda = input->ChildRef(9); + auto& loadLambda = input->ChildRef(10); + auto& lambdaMerge = input->ChildRef(11); + auto& lambdaFinish = input->ChildRef(12); + + if (!EnsureStreamType(*item, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto status = ConvertToLambda(lambdaKeyExtractor, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1)); + status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1)); + status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2)); + status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2)); + status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 3)); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + auto timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp)); + auto itemType = item->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType(); + + if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaKeyExtractor, {itemType}, ctx.Expr)) + { + return IGraphTransformer::TStatus::Error; + } + auto keyType = lambdaKeyExtractor->GetTypeAnn(); + if (!keyType) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) + { + return IGraphTransformer::TStatus::Error; + } + if (!lambdaTimeExtractor->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!EnsureSpecificDataType(*hop, EDataSlot::Interval, ctx.Expr, true)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureSpecificDataType(*interval, EDataSlot::Interval, ctx.Expr, true)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureSpecificDataType(*delay, EDataSlot::Interval, ctx.Expr, true)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!lambdaInit->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + auto stateType = lambdaInit->GetTypeAnn(); + if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!lambdaUpdate->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: " + << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!lambdaMerge->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: " + << *stateType << ", but got: " << *lambdaMerge->GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {keyType, stateType, timeType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!lambdaFinish->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() << + "Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn())); + return IGraphTransformer::TStatus::Error; + } + + if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() << + "Save and load lambdas must be specified at the same time")); + return IGraphTransformer::TStatus::Error; + } + + if (!saveLambda->IsCallable("Void")) { + auto status = ConvertToLambda(saveLambda, ctx.Expr, 1); + status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1)); + if (status.Level != IGraphTransformer::TStatus::Ok) { + return status; + } + + if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!saveLambda->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + auto savedStateType = saveLambda->GetTypeAnn(); + if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!loadLambda->GetTypeAnn()) { + return IGraphTransformer::TStatus::Repeat; + } + + if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, " + << *loadLambda->GetTypeAnn() << " != " << *stateType)); + return IGraphTransformer::TStatus::Error; + } + } + + input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(lambdaFinish->GetTypeAnn())); + return IGraphTransformer::TStatus::Ok; + } + IGraphTransformer::TStatus CombineCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureMinMaxArgsCount(*input, 5U, 6U, ctx.Expr)) { diff --git a/ydb/library/yql/core/type_ann/type_ann_list.h b/ydb/library/yql/core/type_ann/type_ann_list.h index 681400e1b4d..608b813fe8b 100644 --- a/ydb/library/yql/core/type_ann/type_ann_list.h +++ b/ydb/library/yql/core/type_ann/type_ann_list.h @@ -105,7 +105,7 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus WinRowNumberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); - IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListMinWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus ListMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp index 673b1a1be35..f2b793af8d3 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.cpp +++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp @@ -2144,18 +2144,18 @@ IGraphTransformer::TStatus ConvertToLambda(TExprNode::TPtr& node, TExprContext& return IGraphTransformer::TStatus::Repeat; } -bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional) { +bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional) { if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) { YQL_ENSURE(node.Type() == TExprNode::Lambda); ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got lambda")); return false; } - if (allowOptional && node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) { - auto optionalType = node.GetTypeAnn()->Cast<TOptionalExprType>(); - return EnsureSpecificDataType(node.Pos(), *optionalType->GetItemType(), expectedDataSlot, ctx); - } - + if (allowOptional && node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) { + auto optionalType = node.GetTypeAnn()->Cast<TOptionalExprType>(); + return EnsureSpecificDataType(node.Pos(), *optionalType->GetItemType(), expectedDataSlot, ctx); + } + if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got: " << *node.GetTypeAnn())); return false; diff --git a/ydb/library/yql/core/yql_expr_type_annotation.h b/ydb/library/yql/core/yql_expr_type_annotation.h index f81802eae3d..a865aa27cd9 100644 --- a/ydb/library/yql/core/yql_expr_type_annotation.h +++ b/ydb/library/yql/core/yql_expr_type_annotation.h @@ -82,7 +82,7 @@ bool EnsureVariantType(const TExprNode& node, TExprContext& ctx); bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); bool EnsureDataType(const TExprNode& node, TExprContext& ctx); bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); -bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional = false); +bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional = false); bool EnsureSpecificDataType(TPositionHandle position, const TTypeAnnotationNode& type, EDataSlot expectedDataSlot, TExprContext& ctx); bool EnsureStringOrUtf8Type(const TExprNode& node, TExprContext& ctx); bool EnsureStringOrUtf8Type(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx); diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp index 131f758d552..1a1611ec81e 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp @@ -498,11 +498,11 @@ void TDqComputeActorChannels::PassAway() { IActor::PassAway(); } -void TDqComputeActorChannels::SetInputChannelPeer(ui64 channelId, const TActorId& peer) { - TInputChannelState& inputChannel = InCh(channelId); - inputChannel.Peer = peer; -} - +void TDqComputeActorChannels::SetInputChannelPeer(ui64 channelId, const TActorId& peer) { + TInputChannelState& inputChannel = InCh(channelId); + inputChannel.Peer = peer; +} + void TDqComputeActorChannels::SetOutputChannelPeer(ui64 channelId, const TActorId& peer) { TOutputChannelState& outputChannel = OutCh(channelId); outputChannel.Peer = peer; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h index 9b243ef54b5..648b1d8b563 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h @@ -67,7 +67,7 @@ private: public: void SetCheckpointsSupport(); // Finished channels will be polled for checkpoints. - void SetInputChannelPeer(ui64 channelId, const NActors::TActorId& peer); + void SetInputChannelPeer(ui64 channelId, const NActors::TActorId& peer); void SetOutputChannelPeer(ui64 channelId, const NActors::TActorId& peer); bool CanSendChannelData(ui64 channelId); void SendChannelData(NDqProto::TChannelData&& channelData); diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index 713afb7d54a..deaadefb42c 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -5,7 +5,7 @@ #include "dq_compute_actor_checkpoints.h" #include "dq_compute_actor_sinks.h" #include "dq_compute_actor_sources.h" -#include "dq_compute_issues_buffer.h" +#include "dq_compute_issues_buffer.h" #include <ydb/core/scheme/scheme_tabledefs.h> // TODO: TTableId #include <ydb/core/base/kikimr_issue.h> @@ -54,7 +54,7 @@ enum : ui64 { ComputeActorCurrentStateVersion = 2, }; -constexpr ui32 IssuesBufferSize = 16; +constexpr ui32 IssuesBufferSize = 16; template<typename TDerived> class TDqComputeActorBase : public NActors::TActorBootstrapped<TDerived> @@ -257,7 +257,7 @@ protected: virtual void DoExecuteImpl() { auto sourcesState = GetSourcesState(); - PollSourceActors(); + PollSourceActors(); ERunStatus status = TaskRunner->Run(); CA_LOG_D("Resume execution, run status: " << status); @@ -338,9 +338,9 @@ protected: if (status == ERunStatus::PendingInput && ProcessOutputsState.AllOutputsFinished) { CA_LOG_D("All outputs have been finished. Consider finished"); - status = ERunStatus::Finished; - } - + status = ERunStatus::Finished; + } + if (status != ERunStatus::Finished) { // If the incoming channel's buffer was full at the moment when last ChannelDataAck event had been sent, // there will be no attempts to send a new piece of data from the other side of this channel. @@ -670,7 +670,7 @@ protected: struct TInputChannelInfo { ui64 ChannelId; IDqInputChannel::TPtr Channel; - bool HasPeer = false; + bool HasPeer = false; std::optional<NDqProto::TCheckpoint> PendingCheckpoint; const NDqProto::ECheckpointingMode CheckpointingMode; ui64 FreeSpace = 0; @@ -703,11 +703,11 @@ protected: IDqSource::TPtr Source; IDqSourceActor* SourceActor = nullptr; NActors::IActor* Actor = nullptr; - TIssuesBuffer IssuesBuffer; + TIssuesBuffer IssuesBuffer; bool Finished = false; i64 FreeSpace = 1; bool PushStarted = false; - + TSourceInfo(ui64 index) : Index(index), IssuesBuffer(IssuesBufferSize) {} }; @@ -734,11 +734,11 @@ protected: IDqSinkActor* SinkActor = nullptr; NActors::IActor* Actor = nullptr; bool Finished = false; // If sink is in finished state, it receives only checkpoints. - TIssuesBuffer IssuesBuffer; + TIssuesBuffer IssuesBuffer; bool PopStarted = false; i64 SinkActorFreeSpaceBeforeSend = 0; - - TSinkInfo() : IssuesBuffer(IssuesBufferSize) {} + + TSinkInfo() : IssuesBuffer(IssuesBufferSize) {} }; protected: @@ -796,31 +796,31 @@ protected: CA_LOG_D("Received channels info: " << record.ShortDebugString()); for (auto& channelUpdate : record.GetUpdate()) { - TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelUpdate.GetId()); - if (inputChannel && !inputChannel->HasPeer && channelUpdate.GetSrcEndpoint().HasActorId()) { + TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelUpdate.GetId()); + if (inputChannel && !inputChannel->HasPeer && channelUpdate.GetSrcEndpoint().HasActorId()) { auto peer = NActors::ActorIdFromProto(channelUpdate.GetSrcEndpoint().GetActorId()); - - CA_LOG_D("Update input channelId: " << channelUpdate.GetId() << ", peer: " << peer); - - Channels->SetInputChannelPeer(channelUpdate.GetId(), peer); - inputChannel->HasPeer = true; - - continue; - } - + + CA_LOG_D("Update input channelId: " << channelUpdate.GetId() << ", peer: " << peer); + + Channels->SetInputChannelPeer(channelUpdate.GetId(), peer); + inputChannel->HasPeer = true; + + continue; + } + TOutputChannelInfo* outputChannel = OutputChannelsMap.FindPtr(channelUpdate.GetId()); - if (outputChannel && !outputChannel->HasPeer && channelUpdate.GetDstEndpoint().HasActorId()) { + if (outputChannel && !outputChannel->HasPeer && channelUpdate.GetDstEndpoint().HasActorId()) { auto peer = NActors::ActorIdFromProto(channelUpdate.GetDstEndpoint().GetActorId()); CA_LOG_D("Update output channelId: " << channelUpdate.GetId() << ", peer: " << peer); Channels->SetOutputChannelPeer(channelUpdate.GetId(), peer); outputChannel->HasPeer = true; - - continue; + + continue; } - - YQL_ENSURE(inputChannel || outputChannel, "Unknown channelId: " << channelUpdate.GetId() << ", task: " << Task.GetId()); + + YQL_ENSURE(inputChannel || outputChannel, "Unknown channelId: " << channelUpdate.GetId() << ", task: " << Task.GetId()); } DoExecute(); @@ -1171,16 +1171,16 @@ protected: const ui64 i = inputIndex; // Crutch for clang CA_LOG_D("Create source actor for input " << i << " " << inputDesc); std::tie(source.SourceActor, source.Actor) = SourceActorFactory->CreateDqSourceActor( - IDqSourceActorFactory::TArguments{ - .InputDesc = inputDesc, - .InputIndex = inputIndex, - .TxId = TxId, + IDqSourceActorFactory::TArguments{ + .InputDesc = inputDesc, + .InputIndex = inputIndex, + .TxId = TxId, .SecureParams = secureParams, .TaskParams = taskParams, - .Callback = this, + .Callback = this, .TypeEnv = typeEnv, .HolderFactory = holderFactory - }); + }); this->RegisterWithSameMailbox(source.Actor); } if (TaskRunner) { @@ -1195,15 +1195,15 @@ protected: const ui64 i = outputIndex; // Crutch for clang CA_LOG_D("Create sink actor for output " << i << " " << outputDesc); std::tie(sink.SinkActor, sink.Actor) = SinkActorFactory->CreateDqSinkActor( - IDqSinkActorFactory::TArguments { - .OutputDesc = outputDesc, - .OutputIndex = outputIndex, - .TxId = TxId, + IDqSinkActorFactory::TArguments { + .OutputDesc = outputDesc, + .OutputIndex = outputIndex, + .TxId = TxId, .SecureParams = secureParams, - .Callback = this, + .Callback = this, .TypeEnv = typeEnv, .HolderFactory = holderFactory - }); + }); this->RegisterWithSameMailbox(sink.Actor); } } @@ -1242,23 +1242,23 @@ protected: ContinueExecute(); } - void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) override { - if (!isFatal) { - SourcesMap.at(inputIndex).IssuesBuffer.Push(issues); - return; - } - + void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) override { + if (!isFatal) { + SourcesMap.at(inputIndex).IssuesBuffer.Push(issues); + return; + } + TString desc = issues.ToString(); CA_LOG_E("Source[" << inputIndex << "] fatal error: " << desc); InternalError(TIssuesIds::DEFAULT_ERROR, desc); } - void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) override { - if (!isFatal) { - SinksMap.at(outputIndex).IssuesBuffer.Push(issues); - return; - } - + void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) override { + if (!isFatal) { + SinksMap.at(outputIndex).IssuesBuffer.Push(issues); + return; + } + TString desc = issues.ToString(); CA_LOG_E("Sink[" << outputIndex << "] fatal error: " << desc); InternalError(TIssuesIds::DEFAULT_ERROR, desc); @@ -1413,13 +1413,13 @@ private: } } } - + static_cast<TDerived*>(this)->FillExtraStats(dst, last); - + if (last) { BasicStats.reset(); ProfileStats.reset(); - } + } } void ReportStats(TInstant now) { diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp index 2176d57488a..5ae40ef1ab6 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp @@ -3,16 +3,16 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> #include <ydb/library/yql/dq/common/dq_common.h> - + namespace NYql::NDq { -std::pair<IDqSourceActor*, NActors::IActor*> TDqSourceFactory::CreateDqSourceActor(IDqSourceActorFactory::TArguments&& args) const +std::pair<IDqSourceActor*, NActors::IActor*> TDqSourceFactory::CreateDqSourceActor(IDqSourceActorFactory::TArguments&& args) const { - const TString& type = args.InputDesc.GetSource().GetType(); + const TString& type = args.InputDesc.GetSource().GetType(); YQL_ENSURE(!type.empty(), "Attempt to create source actor of empty type"); const TCreatorFunction* creatorFunc = CreatorsByType.FindPtr(type); YQL_ENSURE(creatorFunc, "Unknown type of source actor: \"" << type << "\""); - std::pair<IDqSourceActor*, NActors::IActor*> actor = (*creatorFunc)(std::move(args)); + std::pair<IDqSourceActor*, NActors::IActor*> actor = (*creatorFunc)(std::move(args)); Y_VERIFY(actor.first); Y_VERIFY(actor.second); return actor; @@ -24,13 +24,13 @@ void TDqSourceFactory::Register(const TString& type, TCreatorFunction creator) Y_VERIFY(registered); } -std::pair<IDqSinkActor*, NActors::IActor*> TDqSinkFactory::CreateDqSinkActor(IDqSinkActorFactory::TArguments&& args) const +std::pair<IDqSinkActor*, NActors::IActor*> TDqSinkFactory::CreateDqSinkActor(IDqSinkActorFactory::TArguments&& args) const { - const TString& type = args.OutputDesc.GetSink().GetType(); + const TString& type = args.OutputDesc.GetSink().GetType(); YQL_ENSURE(!type.empty(), "Attempt to create sink actor of empty type"); const TCreatorFunction* creatorFunc = CreatorsByType.FindPtr(type); YQL_ENSURE(creatorFunc, "Unknown type of sink actor: \"" << type << "\""); - std::pair<IDqSinkActor*, NActors::IActor*> actor = (*creatorFunc)(std::move(args)); + std::pair<IDqSinkActor*, NActors::IActor*> actor = (*creatorFunc)(std::move(args)); Y_VERIFY(actor.first); Y_VERIFY(actor.second); return actor; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h index 7b31e818a86..db57796f5d4 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h @@ -4,7 +4,7 @@ #include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> #include <ydb/library/yql/dq/common/dq_common.h> - + #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/utils/yql_panic.h> @@ -17,30 +17,30 @@ concept TCastsToSourceActorPair = std::is_convertible_v<T, std::pair<IDqSourceActor*, NActors::IActor*>>; template <class T, class TProto> -concept TSourceActorCreatorFunc = requires(T f, TProto&& settings, IDqSourceActorFactory::TArguments args) { - { f(std::move(settings), std::move(args)) } -> TCastsToSourceActorPair; +concept TSourceActorCreatorFunc = requires(T f, TProto&& settings, IDqSourceActorFactory::TArguments args) { + { f(std::move(settings), std::move(args)) } -> TCastsToSourceActorPair; }; class TDqSourceFactory : public IDqSourceActorFactory { public: - using TCreatorFunction = std::function<std::pair<IDqSourceActor*, NActors::IActor*>(TArguments&& args)>; + using TCreatorFunction = std::function<std::pair<IDqSourceActor*, NActors::IActor*>(TArguments&& args)>; - std::pair<IDqSourceActor*, NActors::IActor*> CreateDqSourceActor(TArguments&& args) const override; + std::pair<IDqSourceActor*, NActors::IActor*> CreateDqSourceActor(TArguments&& args) const override; void Register(const TString& type, TCreatorFunction creator); template <class TProtoMsg, TSourceActorCreatorFunc<TProtoMsg> TCreatorFunc> void Register(const TString& type, TCreatorFunc creator) { Register(type, - [creator = std::move(creator), type](TArguments&& args) + [creator = std::move(creator), type](TArguments&& args) { - const google::protobuf::Any& settingsAny = args.InputDesc.GetSource().GetSettings(); + const google::protobuf::Any& settingsAny = args.InputDesc.GetSource().GetSettings(); YQL_ENSURE(settingsAny.Is<TProtoMsg>(), "Source \"" << type << "\" settings are expected to have protobuf type " << TProtoMsg::descriptor()->full_name() << ", but got " << settingsAny.type_url()); TProtoMsg settings; YQL_ENSURE(settingsAny.UnpackTo(&settings), "Failed to unpack settings of type \"" << type << "\""); - return creator(std::move(settings), std::move(args)); + return creator(std::move(settings), std::move(args)); }); } @@ -53,30 +53,30 @@ concept TCastsToSinkActorPair = std::is_convertible_v<T, std::pair<IDqSinkActor*, NActors::IActor*>>; template <class T, class TProto> -concept TSinkActorCreatorFunc = requires(T f, TProto&& settings, IDqSinkActorFactory::TArguments&& args) { - { f(std::move(settings), std::move(args)) } -> TCastsToSinkActorPair; +concept TSinkActorCreatorFunc = requires(T f, TProto&& settings, IDqSinkActorFactory::TArguments&& args) { + { f(std::move(settings), std::move(args)) } -> TCastsToSinkActorPair; }; class TDqSinkFactory : public IDqSinkActorFactory { public: - using TCreatorFunction = std::function<std::pair<IDqSinkActor*, NActors::IActor*>(TArguments&& args)>; + using TCreatorFunction = std::function<std::pair<IDqSinkActor*, NActors::IActor*>(TArguments&& args)>; - std::pair<IDqSinkActor*, NActors::IActor*> CreateDqSinkActor(TArguments&& args) const override; + std::pair<IDqSinkActor*, NActors::IActor*> CreateDqSinkActor(TArguments&& args) const override; void Register(const TString& type, TCreatorFunction creator); template <class TProtoMsg, TSinkActorCreatorFunc<TProtoMsg> TCreatorFunc> void Register(const TString& type, TCreatorFunc creator) { Register(type, - [creator = std::move(creator), type](TArguments&& args) + [creator = std::move(creator), type](TArguments&& args) { - const google::protobuf::Any& settingsAny = args.OutputDesc.GetSink().GetSettings(); + const google::protobuf::Any& settingsAny = args.OutputDesc.GetSink().GetSettings(); YQL_ENSURE(settingsAny.Is<TProtoMsg>(), "Sink \"" << type << "\" settings are expected to have protobuf type " << TProtoMsg::descriptor()->full_name() << ", but got " << settingsAny.type_url()); TProtoMsg settings; YQL_ENSURE(settingsAny.UnpackTo(&settings), "Failed to unpack settings of type \"" << type << "\""); - return creator(std::move(settings), std::move(args)); + return creator(std::move(settings), std::move(args)); }); } diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h index e3e74da728e..5c2db2ac8d8 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h @@ -10,7 +10,7 @@ namespace NYql::NDqProto { class TCheckpoint; -class TTaskOutput; +class TTaskOutput; class TSinkState; } // namespace NYql::NDqProto @@ -42,7 +42,7 @@ namespace NYql::NDq { struct IDqSinkActor { struct ICallbacks { // Compute actor virtual void ResumeExecution() = 0; - virtual void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) = 0; + virtual void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) = 0; // Checkpointing virtual void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint& checkpoint) = 0; @@ -74,20 +74,20 @@ struct IDqSinkActor { struct IDqSinkActorFactory : public TThrRefBase { using TPtr = TIntrusivePtr<IDqSinkActorFactory>; - struct TArguments { - const NDqProto::TTaskOutput& OutputDesc; - ui64 OutputIndex; - TTxId TxId; - const THashMap<TString, TString>& SecureParams; - IDqSinkActor::ICallbacks* Callback; - const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv; - const NKikimr::NMiniKQL::THolderFactory& HolderFactory; - }; - + struct TArguments { + const NDqProto::TTaskOutput& OutputDesc; + ui64 OutputIndex; + TTxId TxId; + const THashMap<TString, TString>& SecureParams; + IDqSinkActor::ICallbacks* Callback; + const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv; + const NKikimr::NMiniKQL::THolderFactory& HolderFactory; + }; + // Creates sink actor. // Could throw YQL errors. // IActor* and IDqSinkActor* returned by method must point to the objects with consistent lifetime. - virtual std::pair<IDqSinkActor*, NActors::IActor*> CreateDqSinkActor(TArguments&& args) const = 0; + virtual std::pair<IDqSinkActor*, NActors::IActor*> CreateDqSinkActor(TArguments&& args) const = 0; }; } // namespace NYql::NDq diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h index 647e0645ca4..d2a646cab62 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h @@ -38,7 +38,7 @@ namespace NYql::NDq { struct IDqSourceActor { struct ICallbacks { virtual void OnNewSourceDataArrived(ui64 inputIndex) = 0; - virtual void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) = 0; + virtual void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) = 0; virtual ~ICallbacks() = default; }; @@ -63,21 +63,21 @@ struct IDqSourceActor { struct IDqSourceActorFactory : public TThrRefBase { using TPtr = TIntrusivePtr<IDqSourceActorFactory>; - struct TArguments { - const NDqProto::TTaskInput& InputDesc; - ui64 InputIndex; - TTxId TxId; - const THashMap<TString, TString>& SecureParams; - const THashMap<TString, TString>& TaskParams; - IDqSourceActor::ICallbacks* Callback; - const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv; - const NKikimr::NMiniKQL::THolderFactory& HolderFactory; - }; - + struct TArguments { + const NDqProto::TTaskInput& InputDesc; + ui64 InputIndex; + TTxId TxId; + const THashMap<TString, TString>& SecureParams; + const THashMap<TString, TString>& TaskParams; + IDqSourceActor::ICallbacks* Callback; + const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv; + const NKikimr::NMiniKQL::THolderFactory& HolderFactory; + }; + // Creates source actor. // Could throw YQL errors. // IActor* and IDqSourceActor* returned by method must point to the objects with consistent lifetime. - virtual std::pair<IDqSourceActor*, NActors::IActor*> CreateDqSourceActor(TArguments&& args) const = 0; + virtual std::pair<IDqSourceActor*, NActors::IActor*> CreateDqSourceActor(TArguments&& args) const = 0; }; } // namespace NYql::NDq diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp index 64377cbb8a1..cc13ce4d272 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp @@ -35,13 +35,13 @@ void FillTaskRunnerStats(ui64 taskId, ui32 stageId, const TDqTaskRunnerStats& ta protoBucket->SetValue(snapshot->Value(i)); } } - - for (const auto& stat : taskStats.MkqlStats) { - auto* s = protoTask->MutableMkqlStats()->Add(); - s->SetName(TString(stat.Key.GetName())); - s->SetValue(stat.Value); - s->SetDeriv(stat.Key.IsDeriv()); - } + + for (const auto& stat : taskStats.MkqlStats) { + auto* s = protoTask->MutableMkqlStats()->Add(); + s->SetName(TString(stat.Key.GetName())); + s->SetValue(stat.Value); + s->SetDeriv(stat.Key.IsDeriv()); + } } ui64 firstRowTs = std::numeric_limits<ui64>::max(); diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp index 5d740a0a873..310fb96f91b 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp @@ -1,50 +1,50 @@ -#include "dq_compute_issues_buffer.h" - -#include <algorithm> - -namespace NYql { -namespace NDq { - -TIssuesBuffer::TIssuesBuffer(ui32 capacity) - : Capacity(capacity) - , Issues(capacity) -{ - Y_ENSURE(Capacity); -} - -ui32 TIssuesBuffer::GetAllAddedIssuesCount() const { - return AllAddedIssuesCount; -} - -void TIssuesBuffer::Push(const TIssues& issues) { - AllAddedIssuesCount++; - - auto& issueInfo = Issues.at((HeadIndex + ItemsCount) % Capacity); - issueInfo.Issues = issues; - issueInfo.OccuredAt = TInstant::Now(); - - if (ItemsCount < Capacity) { - ItemsCount++; - return; - } - - HeadIndex = (HeadIndex + 1) % Capacity; -} - -std::vector<TIssueInfo> TIssuesBuffer::Dump() { - std::rotate(Issues.begin(), std::next(Issues.begin(), HeadIndex), Issues.end()); - Issues.resize(ItemsCount); - auto res = std::move(Issues); - Clear(); - return res; -} - -void TIssuesBuffer::Clear() { - Issues.clear(); - Issues.resize(Capacity); - HeadIndex = 0; - ItemsCount = 0; -} - -} -} +#include "dq_compute_issues_buffer.h" + +#include <algorithm> + +namespace NYql { +namespace NDq { + +TIssuesBuffer::TIssuesBuffer(ui32 capacity) + : Capacity(capacity) + , Issues(capacity) +{ + Y_ENSURE(Capacity); +} + +ui32 TIssuesBuffer::GetAllAddedIssuesCount() const { + return AllAddedIssuesCount; +} + +void TIssuesBuffer::Push(const TIssues& issues) { + AllAddedIssuesCount++; + + auto& issueInfo = Issues.at((HeadIndex + ItemsCount) % Capacity); + issueInfo.Issues = issues; + issueInfo.OccuredAt = TInstant::Now(); + + if (ItemsCount < Capacity) { + ItemsCount++; + return; + } + + HeadIndex = (HeadIndex + 1) % Capacity; +} + +std::vector<TIssueInfo> TIssuesBuffer::Dump() { + std::rotate(Issues.begin(), std::next(Issues.begin(), HeadIndex), Issues.end()); + Issues.resize(ItemsCount); + auto res = std::move(Issues); + Clear(); + return res; +} + +void TIssuesBuffer::Clear() { + Issues.clear(); + Issues.resize(Capacity); + HeadIndex = 0; + ItemsCount = 0; +} + +} +} diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h b/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h index a962f61e8af..9bf8f4bf497 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h @@ -1,39 +1,39 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/public/issue/yql_issue.h> - -#include <util/datetime/base.h> - -#include <vector> - -namespace NYql { -namespace NDq { - -struct TIssueInfo { - TIssues Issues; - TInstant OccuredAt; -}; - -// Simple circular buffer -struct TIssuesBuffer { -public: - explicit TIssuesBuffer(ui32 capacity); - - ui32 GetAllAddedIssuesCount() const; - void Push(const TIssues& issues); - std::vector<TIssueInfo> Dump(); - -private: - void Clear(); - -private: - ui32 Capacity; - ui32 AllAddedIssuesCount = 0; - - std::vector<TIssueInfo> Issues; - ui32 HeadIndex = 0; - ui32 ItemsCount = 0; -}; - -} -} + +#include <util/datetime/base.h> + +#include <vector> + +namespace NYql { +namespace NDq { + +struct TIssueInfo { + TIssues Issues; + TInstant OccuredAt; +}; + +// Simple circular buffer +struct TIssuesBuffer { +public: + explicit TIssuesBuffer(ui32 capacity); + + ui32 GetAllAddedIssuesCount() const; + void Push(const TIssues& issues); + std::vector<TIssueInfo> Dump(); + +private: + void Clear(); + +private: + ui32 Capacity; + ui32 AllAddedIssuesCount = 0; + + std::vector<TIssueInfo> Issues; + ui32 HeadIndex = 0; + ui32 ItemsCount = 0; +}; + +} +} diff --git a/ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp b/ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp index f58cf80e68f..64ee3679af1 100644 --- a/ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp +++ b/ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp @@ -1,87 +1,87 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h> - -#include <library/cpp/testing/unittest/registar.h> - -namespace NYql::NDq { - -Y_UNIT_TEST_SUITE(TIssuesBufferTest) { - Y_UNIT_TEST(TestEmpty) { - TIssuesBuffer buffer(5); - - UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 0); - - const auto dumps = buffer.Dump(); - UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 0); - } - - Y_UNIT_TEST(TestSimplePush) { - TIssuesBuffer buffer(5); - - auto issues = TIssues({ TIssue("issue1") }); - buffer.Push(issues); - - UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 1); - - const auto dumps = buffer.Dump(); - UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 1); - UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue1"); - } - - Y_UNIT_TEST(TestPushWithOverflow) { - TIssuesBuffer buffer(2); - - auto issues1 = TIssues({ TIssue("issue1") }); - auto issues2 = TIssues({ TIssue("issue2") }); - auto issues3 = TIssues({ TIssue("issue3") }); - buffer.Push(issues1); - buffer.Push(issues2); - buffer.Push(issues3); - - UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 3); - - const auto dumps = buffer.Dump(); - UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 2); - UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue2"); - UNIT_ASSERT_STRINGS_EQUAL(dumps.at(1).Issues.begin()->Message, "issue3"); - } - - Y_UNIT_TEST(TestSmallBuffer) { - TIssuesBuffer buffer(1); - - auto issues1 = TIssues({ TIssue("issue1") }); - auto issues2 = TIssues({ TIssue("issue2") }); - auto issues3 = TIssues({ TIssue("issue3") }); - buffer.Push(issues1); - buffer.Push(issues2); - buffer.Push(issues3); - - UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 3); - - const auto dumps = buffer.Dump(); - UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 1); - UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue3"); - } - - Y_UNIT_TEST(TestUseAfterDump) { - TIssuesBuffer buffer(2); - - auto issues1 = TIssues({ TIssue("issue1") }); - buffer.Push(issues1); - buffer.Dump(); - - auto issues2 = TIssues({ TIssue("issue2") }); - auto issues3 = TIssues({ TIssue("issue3") }); - auto issues4 = TIssues({ TIssue("issue4") }); - buffer.Push(issues2); - buffer.Push(issues3); - buffer.Push(issues4); - - UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 4); - const auto dumps = buffer.Dump(); - UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 2); - UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue3"); - UNIT_ASSERT_STRINGS_EQUAL(dumps.at(1).Issues.begin()->Message, "issue4"); - } -} - -} // namespace NYql::NDq + +#include <library/cpp/testing/unittest/registar.h> + +namespace NYql::NDq { + +Y_UNIT_TEST_SUITE(TIssuesBufferTest) { + Y_UNIT_TEST(TestEmpty) { + TIssuesBuffer buffer(5); + + UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 0); + + const auto dumps = buffer.Dump(); + UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 0); + } + + Y_UNIT_TEST(TestSimplePush) { + TIssuesBuffer buffer(5); + + auto issues = TIssues({ TIssue("issue1") }); + buffer.Push(issues); + + UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 1); + + const auto dumps = buffer.Dump(); + UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue1"); + } + + Y_UNIT_TEST(TestPushWithOverflow) { + TIssuesBuffer buffer(2); + + auto issues1 = TIssues({ TIssue("issue1") }); + auto issues2 = TIssues({ TIssue("issue2") }); + auto issues3 = TIssues({ TIssue("issue3") }); + buffer.Push(issues1); + buffer.Push(issues2); + buffer.Push(issues3); + + UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 3); + + const auto dumps = buffer.Dump(); + UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue2"); + UNIT_ASSERT_STRINGS_EQUAL(dumps.at(1).Issues.begin()->Message, "issue3"); + } + + Y_UNIT_TEST(TestSmallBuffer) { + TIssuesBuffer buffer(1); + + auto issues1 = TIssues({ TIssue("issue1") }); + auto issues2 = TIssues({ TIssue("issue2") }); + auto issues3 = TIssues({ TIssue("issue3") }); + buffer.Push(issues1); + buffer.Push(issues2); + buffer.Push(issues3); + + UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 3); + + const auto dumps = buffer.Dump(); + UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue3"); + } + + Y_UNIT_TEST(TestUseAfterDump) { + TIssuesBuffer buffer(2); + + auto issues1 = TIssues({ TIssue("issue1") }); + buffer.Push(issues1); + buffer.Dump(); + + auto issues2 = TIssues({ TIssue("issue2") }); + auto issues3 = TIssues({ TIssue("issue3") }); + auto issues4 = TIssues({ TIssue("issue4") }); + buffer.Push(issues2); + buffer.Push(issues3); + buffer.Push(issues4); + + UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 4); + const auto dumps = buffer.Dump(); + UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue3"); + UNIT_ASSERT_STRINGS_EQUAL(dumps.at(1).Issues.begin()->Message, "issue4"); + } +} + +} // namespace NYql::NDq diff --git a/ydb/library/yql/dq/actors/compute/ut/ya.make b/ydb/library/yql/dq/actors/compute/ut/ya.make index 9006163f952..edf1744493c 100644 --- a/ydb/library/yql/dq/actors/compute/ut/ya.make +++ b/ydb/library/yql/dq/actors/compute/ut/ya.make @@ -1,17 +1,17 @@ UNITTEST_FOR(ydb/library/yql/dq/actors/compute) - -OWNER( - d-mokhnatkin - g:yq - g:yql -) - -SRCS( - dq_compute_issues_buffer_ut.cpp -) - -PEERDIR( - library/cpp/testing/unittest -) - -END() + +OWNER( + d-mokhnatkin + g:yq + g:yql +) + +SRCS( + dq_compute_issues_buffer_ut.cpp +) + +PEERDIR( + library/cpp/testing/unittest +) + +END() diff --git a/ydb/library/yql/dq/actors/compute/ya.make b/ydb/library/yql/dq/actors/compute/ya.make index e17b7d97f43..c4b20bd502d 100644 --- a/ydb/library/yql/dq/actors/compute/ya.make +++ b/ydb/library/yql/dq/actors/compute/ya.make @@ -12,7 +12,7 @@ SRCS( dq_compute_actor_checkpoints.cpp dq_compute_actor_io_actors_factory.cpp dq_compute_actor_stats.cpp - dq_compute_issues_buffer.cpp + dq_compute_issues_buffer.cpp retry_queue.cpp ) @@ -32,10 +32,10 @@ PEERDIR( ydb/library/yql/minikql/computation ) -YQL_LAST_ABI_VERSION() +YQL_LAST_ABI_VERSION() -END() +END() -RECURSE_FOR_TESTS( - ut -) +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/dq/actors/protos/dq_stats.proto b/ydb/library/yql/dq/actors/protos/dq_stats.proto index 6df0f4056cb..6b608790305 100644 --- a/ydb/library/yql/dq/actors/protos/dq_stats.proto +++ b/ydb/library/yql/dq/actors/protos/dq_stats.proto @@ -100,12 +100,12 @@ message TDqTableStats { google.protobuf.Any Extra = 100; } -message TDqMkqlStat { - string Name = 1; - int64 Value = 2; - bool Deriv = 3; -} - +message TDqMkqlStat { + string Name = 1; + int64 Value = 2; + bool Deriv = 3; +} + message TDqTaskStats { // basic stats uint64 TaskId = 1; @@ -128,7 +128,7 @@ message TDqTaskStats { uint64 PendingInputTimeUs = 107; // time waiting input data uint64 PendingOutputTimeUs = 108; // time waiting output data uint64 FinishTimeUs = 109; // time in finished state // ComputeCpuTimeUs + PendingInputTimeUs + PendingOutputTimeUs + FinishTimeUs == 100% (or == const in aggregated graphs for several stages/tasks) - repeated TDqMkqlStat MkqlStats = 110; // stats from mkql + repeated TDqMkqlStat MkqlStats = 110; // stats from mkql message THistBucket { double Bound = 1; diff --git a/ydb/library/yql/dq/runtime/dq_output.h b/ydb/library/yql/dq/runtime/dq_output.h index e0014d49821..e793ff5231b 100644 --- a/ydb/library/yql/dq/runtime/dq_output.h +++ b/ydb/library/yql/dq/runtime/dq_output.h @@ -2,12 +2,12 @@ #include <ydb/library/yql/minikql/mkql_node.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <util/generic/ptr.h> namespace NYql { namespace NDqProto { - + class TCheckpoint; class TTaskInput; } // namespace NDqProto @@ -18,19 +18,19 @@ class TUnboxedValue; namespace NDq { -struct TDqOutputStats { +struct TDqOutputStats { // basic stats - ui64 Chunks = 0; - ui64 Bytes = 0; - ui64 RowsIn = 0; - ui64 RowsOut = 0; + ui64 Chunks = 0; + ui64 Bytes = 0; + ui64 RowsIn = 0; + ui64 RowsOut = 0; TInstant FirstRowIn; // profile stats - ui64 MaxMemoryUsage = 0; - ui64 MaxRowsInMemory = 0; -}; - + ui64 MaxMemoryUsage = 0; + ui64 MaxRowsInMemory = 0; +}; + class IDqOutput : public TSimpleRefCount<IDqOutput> { public: using TPtr = TIntrusivePtr<IDqOutput>; @@ -50,10 +50,10 @@ public: [[nodiscard]] virtual bool HasData() const = 0; virtual bool IsFinished() const = 0; - + virtual NKikimr::NMiniKQL::TType* GetOutputType() const = 0; - virtual const TDqOutputStats* GetStats() const = 0; + virtual const TDqOutputStats* GetStats() const = 0; }; } // namespace NDq diff --git a/ydb/library/yql/dq/runtime/dq_output_channel.h b/ydb/library/yql/dq/runtime/dq_output_channel.h index 6d6793ccfd1..f7a4887d2d6 100644 --- a/ydb/library/yql/dq/runtime/dq_output_channel.h +++ b/ydb/library/yql/dq/runtime/dq_output_channel.h @@ -14,7 +14,7 @@ namespace NYql::NDq { -struct TDqOutputChannelStats : TDqOutputStats { +struct TDqOutputChannelStats : TDqOutputStats { ui64 ChannelId = 0; // profile stats diff --git a/ydb/library/yql/dq/runtime/dq_sink.cpp b/ydb/library/yql/dq/runtime/dq_sink.cpp index aca1234ba0a..ff86523ef48 100644 --- a/ydb/library/yql/dq/runtime/dq_sink.cpp +++ b/ydb/library/yql/dq/runtime/dq_sink.cpp @@ -49,32 +49,32 @@ public: void Push(NUdf::TUnboxedValue&& value) override { if (!BasicStats.FirstRowIn) { BasicStats.FirstRowIn = TInstant::Now(); - } - + } + if (ValuesPushed++ % 1000 == 0) { ReestimateRowBytes(value); } Y_VERIFY(EstimatedRowBytes > 0); Values.emplace_back(std::move(value), EstimatedRowBytes); EstimatedStoredBytes += EstimatedRowBytes; - - ReportChunkIn(); + + ReportChunkIn(); } void Push(NDqProto::TCheckpoint&& checkpoint) override { const ui64 bytesSize = checkpoint.ByteSize(); Values.emplace_back(std::move(checkpoint), bytesSize); EstimatedStoredBytes += bytesSize; - - ReportChunkIn(); + + ReportChunkIn(); } void Finish() override { Finished = true; - + if (!BasicStats.FirstRowIn) { BasicStats.FirstRowIn = TInstant::Now(); - } + } } ui64 Pop(NKikimr::NMiniKQL::TUnboxedValueVector& batch, ui64 bytes) override { @@ -99,22 +99,22 @@ public: } Y_VERIFY(EstimatedStoredBytes >= usedBytes); EstimatedStoredBytes -= usedBytes; - - ReportChunkOut(batch.size(), usedBytes); - + + ReportChunkOut(batch.size(), usedBytes); + return usedBytes; } bool Pop(NDqProto::TCheckpoint& checkpoint) override { if (!Values.empty() && std::holds_alternative<NDqProto::TCheckpoint>(Values.front().Value)) { checkpoint = std::move(std::get<NDqProto::TCheckpoint>(Values.front().Value)); - const auto size = Values.front().EstimatedSize; - Y_VERIFY(EstimatedStoredBytes >= size); - EstimatedStoredBytes -= size; + const auto size = Values.front().EstimatedSize; + Y_VERIFY(EstimatedStoredBytes >= size); + EstimatedStoredBytes -= size; Values.pop_front(); - - ReportChunkOut(1, size); - + + ReportChunkOut(1, size); + return true; } return false; @@ -141,10 +141,10 @@ public: return OutputType; } - const TDqSinkStats* GetStats() const override { + const TDqSinkStats* GetStats() const override { return &BasicStats; - } - + } + private: void ReestimateRowBytes(const NUdf::TUnboxedValue& value) { const ui64 valueSize = TDqDataSerializer::EstimateSize(value, OutputType); @@ -158,20 +158,20 @@ private: } } - void ReportChunkIn() { + void ReportChunkIn() { BasicStats.Bytes += EstimatedRowBytes; BasicStats.RowsIn++; if (ProfileStats) { ProfileStats->MaxMemoryUsage = std::max(ProfileStats->MaxMemoryUsage, EstimatedStoredBytes); ProfileStats->MaxRowsInMemory = std::max(ProfileStats->MaxRowsInMemory, Values.size()); - } - } - + } + } + void ReportChunkOut(ui64 rowsCount, ui64 /* usedBytes */) { BasicStats.Chunks++; BasicStats.RowsOut += rowsCount; - } - + } + private: const ui64 OutputIndex; const ui64 MaxStoredBytes; diff --git a/ydb/library/yql/dq/runtime/dq_sink.h b/ydb/library/yql/dq/runtime/dq_sink.h index 6865ed37d2a..63c91c96602 100644 --- a/ydb/library/yql/dq/runtime/dq_sink.h +++ b/ydb/library/yql/dq/runtime/dq_sink.h @@ -7,13 +7,13 @@ namespace NYql::NDq { -struct TDqSinkStats : TDqOutputStats { +struct TDqSinkStats : TDqOutputStats { const ui64 OutputIndex; - - explicit TDqSinkStats(ui64 outputIndex) - : OutputIndex(outputIndex) {} -}; - + + explicit TDqSinkStats(ui64 outputIndex) + : OutputIndex(outputIndex) {} +}; + class IDqSink : public IDqOutput { public: using TPtr = TIntrusivePtr<IDqSink>; @@ -26,8 +26,8 @@ public: // Pop chechpoint. Checkpoints may be taken from sink even after it is finished. [[nodiscard]] virtual bool Pop(NDqProto::TCheckpoint& checkpoint) = 0; - - virtual const TDqSinkStats* GetStats() const = 0; + + virtual const TDqSinkStats* GetStats() const = 0; }; IDqSink::TPtr CreateDqSink(ui64 outputIndex, NKikimr::NMiniKQL::TType* outputType, ui64 maxStoredBytes, diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp index 433c3b4fb6e..10900352c3b 100644 --- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp +++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp @@ -445,8 +445,8 @@ public: TVector<IDqOutput::TPtr> outputs{Reserve(std::max<ui64>(outputDesc.ChannelsSize(), 1))}; if (outputDesc.HasSink()) { - auto sink = CreateDqSink(i, ProgramParsed.OutputItemTypes[i], memoryLimits.ChannelBufferSize, - Settings.CollectProfileStats); + auto sink = CreateDqSink(i, ProgramParsed.OutputItemTypes[i], memoryLimits.ChannelBufferSize, + Settings.CollectProfileStats); auto [_, inserted] = Sinks.emplace(i, sink); Y_VERIFY(inserted); outputs.emplace_back(sink); @@ -523,13 +523,13 @@ public: if (Y_UNLIKELY(CollectProfileStats)) { Stats->ComputeCpuTimeByRun->Collect(RunComputeTime.MilliSeconds()); - - if (ProgramParsed.StatsRegistry) { - Stats->MkqlStats.clear(); - ProgramParsed.StatsRegistry->ForEachStat([this](const TStatKey& key, i64 value) { - Stats->MkqlStats.emplace_back(TMkqlStat{key, value}); - }); - } + + if (ProgramParsed.StatsRegistry) { + Stats->MkqlStats.clear(); + ProgramParsed.StatsRegistry->ForEachStat([this](const TStatKey& key, i64 value) { + Stats->MkqlStats.emplace_back(TMkqlStat{key, value}); + }); + } } if (runStatus == ERunStatus::Finished) { diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.h b/ydb/library/yql/dq/runtime/dq_tasks_runner.h index 46e4e4068e3..a0cc816e91e 100644 --- a/ydb/library/yql/dq/runtime/dq_tasks_runner.h +++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.h @@ -61,11 +61,11 @@ private: TDuration StatusTime[StatusesCount]; }; -struct TMkqlStat { - NKikimr::NMiniKQL::TStatKey Key; - i64 Value = 0; -}; - +struct TMkqlStat { + NKikimr::NMiniKQL::TStatKey Key; + i64 Value = 0; +}; + struct TDqTaskRunnerStats { // basic stats TDuration BuildCpuTime; @@ -83,8 +83,8 @@ struct TDqTaskRunnerStats { THashMap<ui64, const TDqInputChannelStats*> InputChannels; // Channel id -> Channel stats THashMap<ui64, const TDqSourceStats*> Sources; // Input index -> Source stats THashMap<ui64, const TDqOutputChannelStats*> OutputChannels; // Channel id -> Channel stats - - TVector<TMkqlStat> MkqlStats; + + TVector<TMkqlStat> MkqlStats; }; struct TDqTaskRunnerContext { diff --git a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp index 8c7c2ad2a41..fcbd3bb9ecf 100644 --- a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp +++ b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp @@ -151,10 +151,10 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) { if (TDqStageBase::idx_Sinks < stage->ChildrenSize()) { for (const auto& sink: stage->Child(TDqStageBase::idx_Sinks)->Children()) { - sink->SetTypeAnn(resultType); - } - } - + sink->SetTypeAnn(resultType); + } + } + stage->SetTypeAnn(ctx.MakeType<TTupleExprType>(resultTypesTuple)); return TStatus::Ok; } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp index d3ad89e1f19..57599acb641 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp @@ -51,7 +51,7 @@ #include "mkql_lookup.h" #include "mkql_map.h" #include "mkql_map_join.h" -#include "mkql_multihopping.h" +#include "mkql_multihopping.h" #include "mkql_multimap.h" #include "mkql_next_value.h" #include "mkql_now.h" @@ -220,7 +220,7 @@ struct TCallableComputationNodeBuilderFuncMapFiller { {"CombineCore", &WrapCombineCore}, {"GroupingCore", &WrapGroupingCore}, {"HoppingCore", &WrapHoppingCore}, - {"MultiHoppingCore", &WrapMultiHoppingCore}, + {"MultiHoppingCore", &WrapMultiHoppingCore}, {"ToBytes", &WrapToBytes}, {"FromBytes", &WrapFromBytes}, {"NewMTRand", &WrapNewMTRand}, diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp index bea9d861f3a..cb2b13f86d0 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp @@ -1,431 +1,431 @@ -#include "mkql_multihopping.h" -#include "mkql_saveload.h" - +#include "mkql_multihopping.h" +#include "mkql_saveload.h" + #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_stats_registry.h> #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/minikql/watermark_tracker.h> - -#include <util/generic/scope.h> - -namespace NKikimr { -namespace NMiniKQL { - -namespace { - -const TStatKey Hop_NewHopsCount("MultiHop_NewHopsCount", true); -const TStatKey Hop_ThrownEventsCount("MultiHop_ThrownEventsCount", true); -const TStatKey Hop_EmptyTimeCount("MultiHop_EmptyTimeCount", true); + +#include <util/generic/scope.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +const TStatKey Hop_NewHopsCount("MultiHop_NewHopsCount", true); +const TStatKey Hop_ThrownEventsCount("MultiHop_ThrownEventsCount", true); +const TStatKey Hop_EmptyTimeCount("MultiHop_EmptyTimeCount", true); const TStatKey Hop_KeysCount("MultiHop_KeysCount", true); - -constexpr ui32 StateVersion = 1; - -using TEqualsFunc = std::function<bool(NUdf::TUnboxedValuePod, NUdf::TUnboxedValuePod)>; -using THashFunc = std::function<NYql::NUdf::THashType(NUdf::TUnboxedValuePod)>; - + +constexpr ui32 StateVersion = 1; + +using TEqualsFunc = std::function<bool(NUdf::TUnboxedValuePod, NUdf::TUnboxedValuePod)>; +using THashFunc = std::function<NYql::NUdf::THashType(NUdf::TUnboxedValuePod)>; + class TMultiHoppingCoreWrapper : public TStatefulSourceComputationNode<TMultiHoppingCoreWrapper, true> { using TBaseComputation = TStatefulSourceComputationNode<TMultiHoppingCoreWrapper, true>; -public: - using TSelf = TMultiHoppingCoreWrapper; - - class TStreamValue : public TComputationValue<TStreamValue> { - public: - using TBase = TComputationValue<TStreamValue>; - - TStreamValue( - TMemoryUsageInfo* memInfo, - NUdf::TUnboxedValue&& stream, - const TSelf* self, - ui64 hopTime, - ui64 intervalHopCount, - ui64 delayHopCount, - bool dataWatermarks, - TComputationContext& ctx, - const THashFunc& hash, - const TEqualsFunc& equal) - : TBase(memInfo) - , Stream(std::move(stream)) - , Self(self) - , HopTime(hopTime) - , IntervalHopCount(intervalHopCount) - , DelayHopCount(delayHopCount) - , StatesMap(0, hash, equal) - , Ctx(ctx) - { - if (dataWatermarks) { - WatermarkTracker.emplace(TWatermarkTracker(delayHopCount * hopTime, hopTime)); - } - } - - ~TStreamValue() { - ClearState(); - } - - private: - struct TBucket { - NUdf::TUnboxedValue Value; - bool HasValue = false; - }; - - struct TKeyState { - std::vector<TBucket, TMKQLAllocator<TBucket>> Buckets; // circular buffer - ui64 HopIndex; // Start index of current window - - TKeyState(ui64 bucketsCount, ui64 hopIndex) - : Buckets(bucketsCount) - , HopIndex(hopIndex) - {} - - TKeyState(TKeyState&& state) - : Buckets(std::move(state.Buckets)) - , HopIndex(state.HopIndex) - {} - }; - - ui32 GetTraverseCount() const override { - return 1; - } - - NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { - Y_UNUSED(index); - return Stream; - } - - NUdf::TUnboxedValue Save() const override { - MKQL_ENSURE(Ready.empty(), "Inconsistent state to save, not all elements are fetched"); - - TString out; - WriteUi32(out, StateVersion); - WriteUi32(out, StatesMap.size()); - for (const auto& [key, state] : StatesMap) { - WriteUnboxedValue(out, Self->KeyPacker.RefMutableObject(Ctx, false, Self->KeyType), key); - WriteUi64(out, state.HopIndex); - WriteUi32(out, state.Buckets.size()); - for (const auto& bucket : state.Buckets) { - WriteBool(out, bucket.HasValue); - if (bucket.HasValue) { - Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); - if (Self->StateType) { - WriteUnboxedValue(out, Self->StatePacker.RefMutableObject(Ctx, false, Self->StateType), - Self->OutSave->GetValue(Ctx)); - } - } - } - } - - WriteBool(out, Finished); - - auto strRef = NUdf::TStringRef(out.data(), out.size()); - return MakeString(strRef); - } - - void Load(const NUdf::TStringRef& state) override { - TStringBuf in(state.Data(), state.Size()); - - const auto stateVersion = ReadUi32(in); - if (stateVersion == 1) { - const auto statesMapSize = ReadUi32(in); - ClearState(); - StatesMap.reserve(statesMapSize); - for (int i = 0; i < statesMapSize; i++) { - auto key = ReadUnboxedValue(in, Self->KeyPacker.RefMutableObject(Ctx, false, Self->KeyType), Ctx); - const auto hopIndex = ReadUi64(in); - const auto bucketsSize = ReadUi32(in); - - TKeyState keyState(bucketsSize, hopIndex); - for (auto& bucket : keyState.Buckets) { - bucket.HasValue = ReadBool(in); - if (bucket.HasValue) { - if (Self->StateType) { - Self->InLoad->SetValue(Ctx, ReadUnboxedValue(in, Self->StatePacker.RefMutableObject(Ctx, false, Self->StateType), Ctx)); - } - bucket.Value = Self->OutLoad->GetValue(Ctx); - } - } - StatesMap.emplace(key.Release(), std::move(keyState)); - } - - Finished = ReadBool(in); - } else { - THROW yexception() << "Invalid state version " << stateVersion; - } - } - - NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override { - if (!Ready.empty()) { - result = std::move(Ready.front()); - Ready.pop_front(); - return NUdf::EFetchStatus::Ok; - } - - if (Finished) { - return NUdf::EFetchStatus::Finish; - } - - i64 thrownEventsStat = 0; - i64 newHopsStat = 0; - i64 emptyTimeCtStat = 0; - Y_DEFER { - MKQL_ADD_STAT(Ctx.Stats, Hop_ThrownEventsCount, thrownEventsStat); - MKQL_ADD_STAT(Ctx.Stats, Hop_NewHopsCount, newHopsStat); - MKQL_ADD_STAT(Ctx.Stats, Hop_EmptyTimeCount, emptyTimeCtStat); - }; - - for (NUdf::TUnboxedValue item;;) { - if (!Ready.empty()) { - result = std::move(Ready.front()); - Ready.pop_front(); - return NUdf::EFetchStatus::Ok; - } - - const auto status = Stream.Fetch(item); - if (status != NUdf::EFetchStatus::Ok) { - if (status == NUdf::EFetchStatus::Finish) { - CloseOldBuckets(Max<ui64>(), newHopsStat); - Finished = true; - if (!Ready.empty()) { - result = std::move(Ready.front()); - Ready.pop_front(); - return NUdf::EFetchStatus::Ok; - } - } - return status; - } - - Self->Item->SetValue(Ctx, std::move(item)); - auto key = Self->KeyExtract->GetValue(Ctx); - const auto& time = Self->OutTime->GetValue(Ctx); - if (!time) { - ++emptyTimeCtStat; - continue; - } - - const auto ts = time.Get<ui64>(); - const auto hopIndex = ts / HopTime; - auto& keyState = GetOrCreateKeyState(key, hopIndex); - - if (hopIndex < keyState.HopIndex) { - ++thrownEventsStat; - continue; - } - - // Overflow is not possible, because of hopIndex is a product of a division - auto closeBeforeIndex = Max<i64>(hopIndex + 1 - DelayHopCount - IntervalHopCount, 0); - - CloseOldBucketsForKey(key, keyState, closeBeforeIndex, newHopsStat); - - auto& bucket = keyState.Buckets[hopIndex % keyState.Buckets.size()]; - if (!bucket.HasValue) { - bucket.Value = Self->OutInit->GetValue(Ctx); - bucket.HasValue = true; - } else { - Self->Key->SetValue(Ctx, NUdf::TUnboxedValue(key)); - Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); - bucket.Value = Self->OutUpdate->GetValue(Ctx); - } - - if (WatermarkTracker) { - const auto newWatermark = WatermarkTracker->HandleNextEventTime(ts); - if (newWatermark) { - CloseOldBuckets(*newWatermark, newHopsStat); - } - } +public: + using TSelf = TMultiHoppingCoreWrapper; + + class TStreamValue : public TComputationValue<TStreamValue> { + public: + using TBase = TComputationValue<TStreamValue>; + + TStreamValue( + TMemoryUsageInfo* memInfo, + NUdf::TUnboxedValue&& stream, + const TSelf* self, + ui64 hopTime, + ui64 intervalHopCount, + ui64 delayHopCount, + bool dataWatermarks, + TComputationContext& ctx, + const THashFunc& hash, + const TEqualsFunc& equal) + : TBase(memInfo) + , Stream(std::move(stream)) + , Self(self) + , HopTime(hopTime) + , IntervalHopCount(intervalHopCount) + , DelayHopCount(delayHopCount) + , StatesMap(0, hash, equal) + , Ctx(ctx) + { + if (dataWatermarks) { + WatermarkTracker.emplace(TWatermarkTracker(delayHopCount * hopTime, hopTime)); + } + } + + ~TStreamValue() { + ClearState(); + } + + private: + struct TBucket { + NUdf::TUnboxedValue Value; + bool HasValue = false; + }; + + struct TKeyState { + std::vector<TBucket, TMKQLAllocator<TBucket>> Buckets; // circular buffer + ui64 HopIndex; // Start index of current window + + TKeyState(ui64 bucketsCount, ui64 hopIndex) + : Buckets(bucketsCount) + , HopIndex(hopIndex) + {} + + TKeyState(TKeyState&& state) + : Buckets(std::move(state.Buckets)) + , HopIndex(state.HopIndex) + {} + }; + + ui32 GetTraverseCount() const override { + return 1; + } + + NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override { + Y_UNUSED(index); + return Stream; + } + + NUdf::TUnboxedValue Save() const override { + MKQL_ENSURE(Ready.empty(), "Inconsistent state to save, not all elements are fetched"); + + TString out; + WriteUi32(out, StateVersion); + WriteUi32(out, StatesMap.size()); + for (const auto& [key, state] : StatesMap) { + WriteUnboxedValue(out, Self->KeyPacker.RefMutableObject(Ctx, false, Self->KeyType), key); + WriteUi64(out, state.HopIndex); + WriteUi32(out, state.Buckets.size()); + for (const auto& bucket : state.Buckets) { + WriteBool(out, bucket.HasValue); + if (bucket.HasValue) { + Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); + if (Self->StateType) { + WriteUnboxedValue(out, Self->StatePacker.RefMutableObject(Ctx, false, Self->StateType), + Self->OutSave->GetValue(Ctx)); + } + } + } + } + + WriteBool(out, Finished); + + auto strRef = NUdf::TStringRef(out.data(), out.size()); + return MakeString(strRef); + } + + void Load(const NUdf::TStringRef& state) override { + TStringBuf in(state.Data(), state.Size()); + + const auto stateVersion = ReadUi32(in); + if (stateVersion == 1) { + const auto statesMapSize = ReadUi32(in); + ClearState(); + StatesMap.reserve(statesMapSize); + for (int i = 0; i < statesMapSize; i++) { + auto key = ReadUnboxedValue(in, Self->KeyPacker.RefMutableObject(Ctx, false, Self->KeyType), Ctx); + const auto hopIndex = ReadUi64(in); + const auto bucketsSize = ReadUi32(in); + + TKeyState keyState(bucketsSize, hopIndex); + for (auto& bucket : keyState.Buckets) { + bucket.HasValue = ReadBool(in); + if (bucket.HasValue) { + if (Self->StateType) { + Self->InLoad->SetValue(Ctx, ReadUnboxedValue(in, Self->StatePacker.RefMutableObject(Ctx, false, Self->StateType), Ctx)); + } + bucket.Value = Self->OutLoad->GetValue(Ctx); + } + } + StatesMap.emplace(key.Release(), std::move(keyState)); + } + + Finished = ReadBool(in); + } else { + THROW yexception() << "Invalid state version " << stateVersion; + } + } + + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override { + if (!Ready.empty()) { + result = std::move(Ready.front()); + Ready.pop_front(); + return NUdf::EFetchStatus::Ok; + } + + if (Finished) { + return NUdf::EFetchStatus::Finish; + } + + i64 thrownEventsStat = 0; + i64 newHopsStat = 0; + i64 emptyTimeCtStat = 0; + Y_DEFER { + MKQL_ADD_STAT(Ctx.Stats, Hop_ThrownEventsCount, thrownEventsStat); + MKQL_ADD_STAT(Ctx.Stats, Hop_NewHopsCount, newHopsStat); + MKQL_ADD_STAT(Ctx.Stats, Hop_EmptyTimeCount, emptyTimeCtStat); + }; + + for (NUdf::TUnboxedValue item;;) { + if (!Ready.empty()) { + result = std::move(Ready.front()); + Ready.pop_front(); + return NUdf::EFetchStatus::Ok; + } + + const auto status = Stream.Fetch(item); + if (status != NUdf::EFetchStatus::Ok) { + if (status == NUdf::EFetchStatus::Finish) { + CloseOldBuckets(Max<ui64>(), newHopsStat); + Finished = true; + if (!Ready.empty()) { + result = std::move(Ready.front()); + Ready.pop_front(); + return NUdf::EFetchStatus::Ok; + } + } + return status; + } + + Self->Item->SetValue(Ctx, std::move(item)); + auto key = Self->KeyExtract->GetValue(Ctx); + const auto& time = Self->OutTime->GetValue(Ctx); + if (!time) { + ++emptyTimeCtStat; + continue; + } + + const auto ts = time.Get<ui64>(); + const auto hopIndex = ts / HopTime; + auto& keyState = GetOrCreateKeyState(key, hopIndex); + + if (hopIndex < keyState.HopIndex) { + ++thrownEventsStat; + continue; + } + + // Overflow is not possible, because of hopIndex is a product of a division + auto closeBeforeIndex = Max<i64>(hopIndex + 1 - DelayHopCount - IntervalHopCount, 0); + + CloseOldBucketsForKey(key, keyState, closeBeforeIndex, newHopsStat); + + auto& bucket = keyState.Buckets[hopIndex % keyState.Buckets.size()]; + if (!bucket.HasValue) { + bucket.Value = Self->OutInit->GetValue(Ctx); + bucket.HasValue = true; + } else { + Self->Key->SetValue(Ctx, NUdf::TUnboxedValue(key)); + Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); + bucket.Value = Self->OutUpdate->GetValue(Ctx); + } + + if (WatermarkTracker) { + const auto newWatermark = WatermarkTracker->HandleNextEventTime(ts); + if (newWatermark) { + CloseOldBuckets(*newWatermark, newHopsStat); + } + } MKQL_SET_STAT(Ctx.Stats, Hop_KeysCount, StatesMap.size()); - } - } - - TKeyState& GetOrCreateKeyState(NUdf::TUnboxedValue& key, ui64 hopIndex) { - const auto iter = StatesMap.try_emplace( - key, - IntervalHopCount + DelayHopCount, - Max<i64>(hopIndex + 1 - IntervalHopCount, 0) - // For first element we shouldn't forget windows in the past - // Overflow is not possible, because of hopIndex is a product of a division - ); - if (iter.second) { - key.Ref(); - } - return iter.first->second; - } - - // Will return true if key state became empty - bool CloseOldBucketsForKey( - const NUdf::TUnboxedValue& key, - TKeyState& keyState, - const ui64 closeBeforeIndex, // Excluded bound - i64& newHopsStat) - { - auto& bucketsForKey = keyState.Buckets; - - auto stateEmpty = true; - for (auto i = 0; i < bucketsForKey.size(); i++) { - const auto curHopIndex = keyState.HopIndex; - if (curHopIndex >= closeBeforeIndex) { - stateEmpty = false; - break; - } - - TMaybe<NUdf::TUnboxedValue> aggregated; - for (ui64 j = 0; j < IntervalHopCount; j++) { - const auto curBucketIndex = (curHopIndex + j) % bucketsForKey.size(); - const auto& bucket = bucketsForKey[curBucketIndex]; - if (bucket.HasValue) { - if (!aggregated) { // todo: clone - Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); - Self->InLoad->SetValue(Ctx, Self->OutSave->GetValue(Ctx)); - aggregated = Self->OutLoad->GetValue(Ctx); - } else { - Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); - Self->State2->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated)); - aggregated = Self->OutMerge->GetValue(Ctx); - } - } - } - - auto& clearBucket = bucketsForKey[curHopIndex % bucketsForKey.size()]; - clearBucket.Value = NUdf::TUnboxedValue(); - clearBucket.HasValue = false; - - if (aggregated) { - Self->Key->SetValue(Ctx, NUdf::TUnboxedValue(key)); - Self->State->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated)); - // Outer code requires window end time (not start as could be expected) - Self->Time->SetValue(Ctx, NUdf::TUnboxedValuePod((curHopIndex + IntervalHopCount) * HopTime)); - Ready.emplace_back(Self->OutFinish->GetValue(Ctx)); - } - - keyState.HopIndex++; - newHopsStat++; - } - - return stateEmpty; - } - - void CloseOldBuckets(ui64 watermarkTs, i64& newHops) { - const auto watermarkIndex = watermarkTs / HopTime; - EraseNodesIf(StatesMap, [&](auto& iter) { - auto& [key, val] = iter; - const auto keyStateBecameEmpty = CloseOldBucketsForKey(key, val, watermarkIndex + 1 - IntervalHopCount, newHops); - if (keyStateBecameEmpty) { - key.UnRef(); - } - return keyStateBecameEmpty; - }); - return; - } - - void ClearState() { - EraseNodesIf(StatesMap, [&](auto& iter) { - iter.first.UnRef(); - return true; - }); - StatesMap.rehash(0); - } - - const NUdf::TUnboxedValue Stream; - const TSelf *const Self; - - const ui64 HopTime; - const ui64 IntervalHopCount; - const ui64 DelayHopCount; - - using TStatesMap = std::unordered_map< - NUdf::TUnboxedValuePod, TKeyState, - THashFunc, TEqualsFunc, - TMKQLAllocator<std::pair<const NUdf::TUnboxedValuePod, TKeyState>>>; - - TStatesMap StatesMap; // Map of states for each key - std::deque<NUdf::TUnboxedValue> Ready; // buffer for fetching results - bool Finished = false; - - TComputationContext& Ctx; - std::optional<TWatermarkTracker> WatermarkTracker; - }; - - TMultiHoppingCoreWrapper( - TComputationMutables& mutables, - IComputationNode* stream, - IComputationExternalNode* item, - IComputationExternalNode* key, - IComputationExternalNode* state, - IComputationExternalNode* state2, - IComputationExternalNode* time, - IComputationExternalNode* inSave, - IComputationExternalNode* inLoad, - IComputationNode* keyExtract, - IComputationNode* outTime, - IComputationNode* outInit, - IComputationNode* outUpdate, - IComputationNode* outSave, - IComputationNode* outLoad, - IComputationNode* outMerge, - IComputationNode* outFinish, - IComputationNode* hop, - IComputationNode* interval, - IComputationNode* delay, - IComputationNode* dataWatermarks, - TType* keyType, - TType* stateType) - : TBaseComputation(mutables) - , Stream(stream) - , Item(item) - , Key(key) - , State(state) - , State2(state2) - , Time(time) - , InSave(inSave) - , InLoad(inLoad) - , KeyExtract(keyExtract) - , OutTime(outTime) - , OutInit(outInit) - , OutUpdate(outUpdate) - , OutSave(outSave) - , OutLoad(outLoad) - , OutMerge(outMerge) - , OutFinish(outFinish) - , Hop(hop) - , Interval(interval) - , Delay(delay) - , DataWatermarks(dataWatermarks) - , KeyType(keyType) - , StateType(stateType) - , KeyPacker(mutables) - , StatePacker(mutables) - , KeyTypes() - , IsTuple() - { - Stateless = false; - - bool encoded; - GetDictionaryKeyTypes(keyType, KeyTypes, IsTuple, encoded); - Y_VERIFY(!encoded, "TODO"); - } - + } + } + + TKeyState& GetOrCreateKeyState(NUdf::TUnboxedValue& key, ui64 hopIndex) { + const auto iter = StatesMap.try_emplace( + key, + IntervalHopCount + DelayHopCount, + Max<i64>(hopIndex + 1 - IntervalHopCount, 0) + // For first element we shouldn't forget windows in the past + // Overflow is not possible, because of hopIndex is a product of a division + ); + if (iter.second) { + key.Ref(); + } + return iter.first->second; + } + + // Will return true if key state became empty + bool CloseOldBucketsForKey( + const NUdf::TUnboxedValue& key, + TKeyState& keyState, + const ui64 closeBeforeIndex, // Excluded bound + i64& newHopsStat) + { + auto& bucketsForKey = keyState.Buckets; + + auto stateEmpty = true; + for (auto i = 0; i < bucketsForKey.size(); i++) { + const auto curHopIndex = keyState.HopIndex; + if (curHopIndex >= closeBeforeIndex) { + stateEmpty = false; + break; + } + + TMaybe<NUdf::TUnboxedValue> aggregated; + for (ui64 j = 0; j < IntervalHopCount; j++) { + const auto curBucketIndex = (curHopIndex + j) % bucketsForKey.size(); + const auto& bucket = bucketsForKey[curBucketIndex]; + if (bucket.HasValue) { + if (!aggregated) { // todo: clone + Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); + Self->InLoad->SetValue(Ctx, Self->OutSave->GetValue(Ctx)); + aggregated = Self->OutLoad->GetValue(Ctx); + } else { + Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value)); + Self->State2->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated)); + aggregated = Self->OutMerge->GetValue(Ctx); + } + } + } + + auto& clearBucket = bucketsForKey[curHopIndex % bucketsForKey.size()]; + clearBucket.Value = NUdf::TUnboxedValue(); + clearBucket.HasValue = false; + + if (aggregated) { + Self->Key->SetValue(Ctx, NUdf::TUnboxedValue(key)); + Self->State->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated)); + // Outer code requires window end time (not start as could be expected) + Self->Time->SetValue(Ctx, NUdf::TUnboxedValuePod((curHopIndex + IntervalHopCount) * HopTime)); + Ready.emplace_back(Self->OutFinish->GetValue(Ctx)); + } + + keyState.HopIndex++; + newHopsStat++; + } + + return stateEmpty; + } + + void CloseOldBuckets(ui64 watermarkTs, i64& newHops) { + const auto watermarkIndex = watermarkTs / HopTime; + EraseNodesIf(StatesMap, [&](auto& iter) { + auto& [key, val] = iter; + const auto keyStateBecameEmpty = CloseOldBucketsForKey(key, val, watermarkIndex + 1 - IntervalHopCount, newHops); + if (keyStateBecameEmpty) { + key.UnRef(); + } + return keyStateBecameEmpty; + }); + return; + } + + void ClearState() { + EraseNodesIf(StatesMap, [&](auto& iter) { + iter.first.UnRef(); + return true; + }); + StatesMap.rehash(0); + } + + const NUdf::TUnboxedValue Stream; + const TSelf *const Self; + + const ui64 HopTime; + const ui64 IntervalHopCount; + const ui64 DelayHopCount; + + using TStatesMap = std::unordered_map< + NUdf::TUnboxedValuePod, TKeyState, + THashFunc, TEqualsFunc, + TMKQLAllocator<std::pair<const NUdf::TUnboxedValuePod, TKeyState>>>; + + TStatesMap StatesMap; // Map of states for each key + std::deque<NUdf::TUnboxedValue> Ready; // buffer for fetching results + bool Finished = false; + + TComputationContext& Ctx; + std::optional<TWatermarkTracker> WatermarkTracker; + }; + + TMultiHoppingCoreWrapper( + TComputationMutables& mutables, + IComputationNode* stream, + IComputationExternalNode* item, + IComputationExternalNode* key, + IComputationExternalNode* state, + IComputationExternalNode* state2, + IComputationExternalNode* time, + IComputationExternalNode* inSave, + IComputationExternalNode* inLoad, + IComputationNode* keyExtract, + IComputationNode* outTime, + IComputationNode* outInit, + IComputationNode* outUpdate, + IComputationNode* outSave, + IComputationNode* outLoad, + IComputationNode* outMerge, + IComputationNode* outFinish, + IComputationNode* hop, + IComputationNode* interval, + IComputationNode* delay, + IComputationNode* dataWatermarks, + TType* keyType, + TType* stateType) + : TBaseComputation(mutables) + , Stream(stream) + , Item(item) + , Key(key) + , State(state) + , State2(state2) + , Time(time) + , InSave(inSave) + , InLoad(inLoad) + , KeyExtract(keyExtract) + , OutTime(outTime) + , OutInit(outInit) + , OutUpdate(outUpdate) + , OutSave(outSave) + , OutLoad(outLoad) + , OutMerge(outMerge) + , OutFinish(outFinish) + , Hop(hop) + , Interval(interval) + , Delay(delay) + , DataWatermarks(dataWatermarks) + , KeyType(keyType) + , StateType(stateType) + , KeyPacker(mutables) + , StatePacker(mutables) + , KeyTypes() + , IsTuple() + { + Stateless = false; + + bool encoded; + GetDictionaryKeyTypes(keyType, KeyTypes, IsTuple, encoded); + Y_VERIFY(!encoded, "TODO"); + } + NUdf::TUnboxedValuePod CreateStream(TComputationContext& ctx) const { - const auto hopTime = Hop->GetValue(ctx).Get<i64>(); - const auto interval = Interval->GetValue(ctx).Get<i64>(); - const auto delay = Delay->GetValue(ctx).Get<i64>(); - const auto dataWatermarks = DataWatermarks->GetValue(ctx).Get<bool>(); - - // TODO: move checks from here - MKQL_ENSURE(hopTime > 0, "hop must be positive"); - MKQL_ENSURE(interval >= hopTime, "interval should be greater or equal to hop"); - MKQL_ENSURE(delay >= hopTime, "delay should be greater or equal to hop"); - - const auto intervalHopCount = interval / hopTime; - const auto delayHopCount = delay / hopTime; - - MKQL_ENSURE(intervalHopCount <= 100000, "too many hops in interval"); - MKQL_ENSURE(delayHopCount <= 100000, "too many hops in delay"); - - return ctx.HolderFactory.Create<TStreamValue>(Stream->GetValue(ctx), this, (ui64)hopTime, - (ui64)intervalHopCount, (ui64)delayHopCount, - dataWatermarks, ctx, - TValueHasher(KeyTypes, IsTuple), - TValueEqual(KeyTypes, IsTuple)); - } - + const auto hopTime = Hop->GetValue(ctx).Get<i64>(); + const auto interval = Interval->GetValue(ctx).Get<i64>(); + const auto delay = Delay->GetValue(ctx).Get<i64>(); + const auto dataWatermarks = DataWatermarks->GetValue(ctx).Get<bool>(); + + // TODO: move checks from here + MKQL_ENSURE(hopTime > 0, "hop must be positive"); + MKQL_ENSURE(interval >= hopTime, "interval should be greater or equal to hop"); + MKQL_ENSURE(delay >= hopTime, "delay should be greater or equal to hop"); + + const auto intervalHopCount = interval / hopTime; + const auto delayHopCount = delay / hopTime; + + MKQL_ENSURE(intervalHopCount <= 100000, "too many hops in interval"); + MKQL_ENSURE(delayHopCount <= 100000, "too many hops in delay"); + + return ctx.HolderFactory.Create<TStreamValue>(Stream->GetValue(ctx), this, (ui64)hopTime, + (ui64)intervalHopCount, (ui64)delayHopCount, + dataWatermarks, ctx, + TValueHasher(KeyTypes, IsTuple), + TValueEqual(KeyTypes, IsTuple)); + } + NUdf::TUnboxedValue GetValue(TComputationContext& compCtx) const override { NUdf::TUnboxedValue& valueRef = ValueRef(compCtx); if (valueRef.IsInvalid()) { @@ -441,115 +441,115 @@ public: return valueRef; } -private: - void RegisterDependencies() const final { - DependsOn(Stream); - Own(Item); - Own(Key); - Own(State); - Own(State2); - Own(Time); - Own(InSave); - Own(InLoad); - DependsOn(KeyExtract); - DependsOn(OutTime); - DependsOn(OutInit); - DependsOn(OutUpdate); - DependsOn(OutSave); - DependsOn(OutLoad); - DependsOn(OutMerge); - DependsOn(OutFinish); - DependsOn(Hop); - DependsOn(Interval); - DependsOn(Delay); - DependsOn(DataWatermarks); - } - - IComputationNode* const Stream; - - IComputationExternalNode* const Item; - IComputationExternalNode* const Key; - IComputationExternalNode* const State; - IComputationExternalNode* const State2; - IComputationExternalNode* const Time; - IComputationExternalNode* const InSave; - IComputationExternalNode* const InLoad; - - IComputationNode* const KeyExtract; - IComputationNode* const OutTime; - IComputationNode* const OutInit; - IComputationNode* const OutUpdate; - IComputationNode* const OutSave; - IComputationNode* const OutLoad; - IComputationNode* const OutMerge; - IComputationNode* const OutFinish; - - IComputationNode* const Hop; - IComputationNode* const Interval; - IComputationNode* const Delay; - IComputationNode* const DataWatermarks; - - TType* const KeyType; - TType* const StateType; - TMutableObjectOverBoxedValue<TValuePackerBoxed> KeyPacker; - TMutableObjectOverBoxedValue<TValuePackerBoxed> StatePacker; - - TKeyTypes KeyTypes; - bool IsTuple; -}; - -} - -IComputationNode* WrapMultiHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 20, "Expected 20 args"); - - auto hasSaveLoad = !callable.GetInput(12).GetStaticType()->IsVoid(); - - IComputationExternalNode* inSave = nullptr; - IComputationNode* outSave = nullptr; - IComputationExternalNode* inLoad = nullptr; - IComputationNode* outLoad = nullptr; - - auto streamType = callable.GetInput(0).GetStaticType(); - MKQL_ENSURE(streamType->IsStream(), "Expected stream"); - - const auto keyType = callable.GetInput(8).GetStaticType(); - - auto stream = LocateNode(ctx.NodeLocator, callable, 0); - - auto keyExtract = LocateNode(ctx.NodeLocator, callable, 8); - auto outTime = LocateNode(ctx.NodeLocator, callable, 9); - auto outInit = LocateNode(ctx.NodeLocator, callable, 10); - auto outUpdate = LocateNode(ctx.NodeLocator, callable, 11); - if (hasSaveLoad) { - outSave = LocateNode(ctx.NodeLocator, callable, 12); - outLoad = LocateNode(ctx.NodeLocator, callable, 13); - } - auto outMerge = LocateNode(ctx.NodeLocator, callable, 14); - auto outFinish = LocateNode(ctx.NodeLocator, callable, 15); - - auto hop = LocateNode(ctx.NodeLocator, callable, 16); - auto interval = LocateNode(ctx.NodeLocator, callable, 17); - auto delay = LocateNode(ctx.NodeLocator, callable, 18); - auto dataWatermarks = LocateNode(ctx.NodeLocator, callable, 19); - - auto item = LocateExternalNode(ctx.NodeLocator, callable, 1); - auto key = LocateExternalNode(ctx.NodeLocator, callable, 2); - auto state = LocateExternalNode(ctx.NodeLocator, callable, 3); - auto state2 = LocateExternalNode(ctx.NodeLocator, callable, 4); - auto time = LocateExternalNode(ctx.NodeLocator, callable, 5); - if (hasSaveLoad) { - inSave = LocateExternalNode(ctx.NodeLocator, callable, 6); - inLoad = LocateExternalNode(ctx.NodeLocator, callable, 7); - } - - auto stateType = hasSaveLoad ? callable.GetInput(12).GetStaticType() : nullptr; - - return new TMultiHoppingCoreWrapper(ctx.Mutables, - stream, item, key, state, state2, time, inSave, inLoad, keyExtract, - outTime, outInit, outUpdate, outSave, outLoad, outMerge, outFinish, - hop, interval, delay, dataWatermarks, keyType, stateType); -} - -} -} +private: + void RegisterDependencies() const final { + DependsOn(Stream); + Own(Item); + Own(Key); + Own(State); + Own(State2); + Own(Time); + Own(InSave); + Own(InLoad); + DependsOn(KeyExtract); + DependsOn(OutTime); + DependsOn(OutInit); + DependsOn(OutUpdate); + DependsOn(OutSave); + DependsOn(OutLoad); + DependsOn(OutMerge); + DependsOn(OutFinish); + DependsOn(Hop); + DependsOn(Interval); + DependsOn(Delay); + DependsOn(DataWatermarks); + } + + IComputationNode* const Stream; + + IComputationExternalNode* const Item; + IComputationExternalNode* const Key; + IComputationExternalNode* const State; + IComputationExternalNode* const State2; + IComputationExternalNode* const Time; + IComputationExternalNode* const InSave; + IComputationExternalNode* const InLoad; + + IComputationNode* const KeyExtract; + IComputationNode* const OutTime; + IComputationNode* const OutInit; + IComputationNode* const OutUpdate; + IComputationNode* const OutSave; + IComputationNode* const OutLoad; + IComputationNode* const OutMerge; + IComputationNode* const OutFinish; + + IComputationNode* const Hop; + IComputationNode* const Interval; + IComputationNode* const Delay; + IComputationNode* const DataWatermarks; + + TType* const KeyType; + TType* const StateType; + TMutableObjectOverBoxedValue<TValuePackerBoxed> KeyPacker; + TMutableObjectOverBoxedValue<TValuePackerBoxed> StatePacker; + + TKeyTypes KeyTypes; + bool IsTuple; +}; + +} + +IComputationNode* WrapMultiHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 20, "Expected 20 args"); + + auto hasSaveLoad = !callable.GetInput(12).GetStaticType()->IsVoid(); + + IComputationExternalNode* inSave = nullptr; + IComputationNode* outSave = nullptr; + IComputationExternalNode* inLoad = nullptr; + IComputationNode* outLoad = nullptr; + + auto streamType = callable.GetInput(0).GetStaticType(); + MKQL_ENSURE(streamType->IsStream(), "Expected stream"); + + const auto keyType = callable.GetInput(8).GetStaticType(); + + auto stream = LocateNode(ctx.NodeLocator, callable, 0); + + auto keyExtract = LocateNode(ctx.NodeLocator, callable, 8); + auto outTime = LocateNode(ctx.NodeLocator, callable, 9); + auto outInit = LocateNode(ctx.NodeLocator, callable, 10); + auto outUpdate = LocateNode(ctx.NodeLocator, callable, 11); + if (hasSaveLoad) { + outSave = LocateNode(ctx.NodeLocator, callable, 12); + outLoad = LocateNode(ctx.NodeLocator, callable, 13); + } + auto outMerge = LocateNode(ctx.NodeLocator, callable, 14); + auto outFinish = LocateNode(ctx.NodeLocator, callable, 15); + + auto hop = LocateNode(ctx.NodeLocator, callable, 16); + auto interval = LocateNode(ctx.NodeLocator, callable, 17); + auto delay = LocateNode(ctx.NodeLocator, callable, 18); + auto dataWatermarks = LocateNode(ctx.NodeLocator, callable, 19); + + auto item = LocateExternalNode(ctx.NodeLocator, callable, 1); + auto key = LocateExternalNode(ctx.NodeLocator, callable, 2); + auto state = LocateExternalNode(ctx.NodeLocator, callable, 3); + auto state2 = LocateExternalNode(ctx.NodeLocator, callable, 4); + auto time = LocateExternalNode(ctx.NodeLocator, callable, 5); + if (hasSaveLoad) { + inSave = LocateExternalNode(ctx.NodeLocator, callable, 6); + inLoad = LocateExternalNode(ctx.NodeLocator, callable, 7); + } + + auto stateType = hasSaveLoad ? callable.GetInput(12).GetStaticType() : nullptr; + + return new TMultiHoppingCoreWrapper(ctx.Mutables, + stream, item, key, state, state2, time, inSave, inLoad, keyExtract, + outTime, outInit, outUpdate, outSave, outLoad, outMerge, outFinish, + hop, interval, delay, dataWatermarks, keyType, stateType); +} + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h index f0c7dbb6971..457b2ebe55a 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> - -namespace NKikimr { -namespace NMiniKQL { - -IComputationNode* WrapMultiHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx); - -} -} + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapMultiHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx); + +} +} diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp index 2947e9516a3..6819dc77187 100644 --- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp @@ -1,300 +1,300 @@ -#include <ydb/library/yql/minikql/mkql_node.h> -#include <ydb/library/yql/minikql/mkql_node_cast.h> -#include <ydb/library/yql/minikql/mkql_program_builder.h> -#include <ydb/library/yql/minikql/mkql_function_registry.h> -#include <ydb/library/yql/minikql/computation/mkql_computation_node.h> -#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> -#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h> -#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> -#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> - -#include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { -namespace NMiniKQL { - -namespace { - TIntrusivePtr<IRandomProvider> CreateRandomProvider() { - return CreateDeterministicRandomProvider(1); - } - - TIntrusivePtr<ITimeProvider> CreateTimeProvider() { - return CreateDeterministicTimeProvider(10000000); - } - - TComputationNodeFactory GetAuxCallableFactory() { - return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { - if (callable.GetType()->GetName() == "OneYieldStream") { - return new TExternalComputationNode(ctx.Mutables); - } - - return GetBuiltinFactory()(callable, ctx); - }; - } - - struct TSetup { - TSetup(TScopedAlloc& alloc) - : Alloc(alloc) - { - FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); - RandomProvider = CreateRandomProvider(); - TimeProvider = CreateTimeProvider(); - - Env.Reset(new TTypeEnvironment(Alloc)); - PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); - } - - THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { - Explorer.Walk(pgm.GetNode(), *Env); - TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(), - FunctionRegistry.Get(), - NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi); - Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); - TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); - return Pattern->Clone(compOpts); - } - - TIntrusivePtr<IFunctionRegistry> FunctionRegistry; - TIntrusivePtr<IRandomProvider> RandomProvider; - TIntrusivePtr<ITimeProvider> TimeProvider; - - TScopedAlloc& Alloc; - THolder<TTypeEnvironment> Env; - THolder<TProgramBuilder> PgmBuilder; - - TExploringNodeVisitor Explorer; - IComputationPattern::TPtr Pattern; - }; - - struct TStreamWithYield : public NUdf::TBoxedValue { - TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index) - : Items(items) - , YieldPos(yieldPos) - , Index(index) - {} - - private: - TUnboxedValueVector Items; - ui32 YieldPos; - ui32 Index; - - ui32 GetTraverseCount() const override { - return 0; - } - - NUdf::TUnboxedValue Save() const override { - return NUdf::TUnboxedValue::Zero(); - } - - void Load(const NUdf::TStringRef& state) override { - Y_UNUSED(state); - } - - NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final { - if (Index >= Items.size()) { - return NUdf::EFetchStatus::Finish; - } - if (Index == YieldPos) { - return NUdf::EFetchStatus::Yield; - } - result = Items[Index++]; - return NUdf::EFetchStatus::Ok; - } - }; - - THolder<IComputationGraph> BuildGraph(TSetup& setup, const std::vector<std::tuple<ui32, i64, ui32>> items, - ui32 yieldPos, ui32 startIndex, bool dataWatermarks) { - TProgramBuilder& pgmBuilder = *setup.PgmBuilder; - - auto structType = pgmBuilder.NewEmptyStructType(); - structType = pgmBuilder.NewStructType(structType, "key", - pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); - structType = pgmBuilder.NewStructType(structType, "time", - pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); - structType = pgmBuilder.NewStructType(structType, "sum", - pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); - auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key"); - auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); - auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); - - auto inStreamType = pgmBuilder.NewStreamType(structType); - - TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType); - auto streamNode = inStream.Build(); - - ui64 hop = 10, interval = 30, delay = 20; - - auto pgmReturn = pgmBuilder.MultiHoppingCore( - TRuntimeNode(streamNode, false), - [&](TRuntimeNode item) { // keyExtractor - return pgmBuilder.Member(item, "key"); - }, - [&](TRuntimeNode item) { // timeExtractor - return pgmBuilder.Member(item, "time"); - }, - [&](TRuntimeNode item) { // init - std::vector<std::pair<std::string_view, TRuntimeNode>> members; - members.emplace_back("sum", pgmBuilder.Member(item, "sum")); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode item, TRuntimeNode state) { // update - auto add = pgmBuilder.AggrAdd( - pgmBuilder.Member(item, "sum"), - pgmBuilder.Member(state, "sum")); - std::vector<std::pair<std::string_view, TRuntimeNode>> members; - members.emplace_back("sum", add); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode state) { // save - return pgmBuilder.Member(state, "sum"); - }, - [&](TRuntimeNode savedState) { // load - std::vector<std::pair<std::string_view, TRuntimeNode>> members; - members.emplace_back("sum", savedState); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode state1, TRuntimeNode state2) { // merge - auto add = pgmBuilder.AggrAdd( - pgmBuilder.Member(state1, "sum"), - pgmBuilder.Member(state2, "sum")); - std::vector<std::pair<std::string_view, TRuntimeNode>> members; - members.emplace_back("sum", add); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish - Y_UNUSED(time); - std::vector<std::pair<std::string_view, TRuntimeNode>> members; - members.emplace_back("key", key); - members.emplace_back("sum", pgmBuilder.Member(state, "sum")); - return pgmBuilder.NewStruct(members); - }, - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay - pgmBuilder.NewDataLiteral<bool>(dataWatermarks) // dataWatermarks - ); - - auto graph = setup.BuildGraph(pgmReturn, {streamNode}); - - TUnboxedValueVector streamItems; - for (size_t i = 0; i < items.size(); ++i) { - NUdf::TUnboxedValue* itemsPtr; - auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr); - itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(std::get<0>(items[i])); - itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(std::get<1>(items[i])); - itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(std::get<2>(items[i])); - streamItems.push_back(std::move(structValues)); - } - - auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); - graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); - return graph; - } -} - -Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingSaveLoadTest) { - void TestWithSaveLoadImpl( - const std::vector<std::tuple<ui32, i64, ui32>> input, - const std::vector<std::tuple<ui32, ui32>> expected, - bool withTraverse, - bool dataWatermarks) - { - TScopedAlloc alloc; - - for (ui32 yieldPos = 0; yieldPos < input.size(); ++yieldPos) { - std::vector<std::tuple<ui32, ui32>> result; - - TSetup setup1(alloc); - auto graph1 = BuildGraph(setup1, input, yieldPos, 0, dataWatermarks); - auto root1 = graph1->GetValue(); - - NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; - while (status == NUdf::EFetchStatus::Ok) { - NUdf::TUnboxedValue val; - status = root1.Fetch(val); - if (status == NUdf::EFetchStatus::Ok) { - result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>()); - } - } - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); - - TString graphState; - if (withTraverse) { - SaveGraphState(&root1, 1, 0ULL, graphState); - } else { - graphState = graph1->SaveGraphState(); - } - - TSetup setup2(alloc); - auto graph2 = BuildGraph(setup2, input, -1, yieldPos, dataWatermarks); - NUdf::TUnboxedValue root2; - if (withTraverse) { - root2 = graph2->GetValue(); - LoadGraphState(&root2, 1, 0ULL, graphState); - } else { - graph2->LoadGraphState(graphState); - root2 = graph2->GetValue(); - } - - status = NUdf::EFetchStatus::Ok; - while (status == NUdf::EFetchStatus::Ok) { - NUdf::TUnboxedValue val; - status = root2.Fetch(val); - if (status == NUdf::EFetchStatus::Ok) { - result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>()); - } - } - UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); - - auto sortedExpected = expected; - std::sort(result.begin(), result.end()); - std::sort(sortedExpected.begin(), sortedExpected.end()); - UNIT_ASSERT_EQUAL(result, sortedExpected); - } - } - - const std::vector<std::tuple<ui32, i64, ui32>> input1 = { - // Group; Time; Value - {2, 1, 2}, - {1, 1, 2}, - {2, 2, 3}, - {1, 2, 3}, - {2, 15, 4}, - {1, 15, 4}, - {2, 23, 6}, - {1, 23, 6}, - {2, 24, 5}, - {1, 24, 5}, - {2, 25, 7}, - {1, 25, 7}, - {2, 40, 2}, - {1, 40, 2}, - {2, 47, 1}, - {1, 47, 1}, - {2, 51, 6}, - {1, 51, 6}, - {2, 59, 2}, - {1, 59, 2}, - {2, 85, 8}, - {1, 85, 8} - }; - - const std::vector<std::tuple<ui32, ui32>> expected = { - {1, 8}, {1, 8}, {1, 8}, {1, 8}, - {1, 11}, {1, 11}, {1, 21}, {1, 22}, - {1, 27}, - {2, 8}, {2, 8}, {2, 8}, {2, 8}, - {2, 11}, {2, 11}, {2, 21}, - {2, 22}, {2, 27}}; - - Y_UNIT_TEST(Test1) { - TestWithSaveLoadImpl(input1, expected, true, false); - } - - Y_UNIT_TEST(Test2) { - TestWithSaveLoadImpl(input1, expected, false, false); - } -} - -} // namespace NMiniKQL -} // namespace NKikimr +#include <ydb/library/yql/minikql/mkql_node.h> +#include <ydb/library/yql/minikql/mkql_node_cast.h> +#include <ydb/library/yql/minikql/mkql_program_builder.h> +#include <ydb/library/yql/minikql/mkql_function_registry.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h> +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + TIntrusivePtr<IRandomProvider> CreateRandomProvider() { + return CreateDeterministicRandomProvider(1); + } + + TIntrusivePtr<ITimeProvider> CreateTimeProvider() { + return CreateDeterministicTimeProvider(10000000); + } + + TComputationNodeFactory GetAuxCallableFactory() { + return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "OneYieldStream") { + return new TExternalComputationNode(ctx.Mutables); + } + + return GetBuiltinFactory()(callable, ctx); + }; + } + + struct TSetup { + TSetup(TScopedAlloc& alloc) + : Alloc(alloc) + { + FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + RandomProvider = CreateRandomProvider(); + TimeProvider = CreateTimeProvider(); + + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + + THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(), + FunctionRegistry.Get(), + NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); + return Pattern->Clone(compOpts); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + + TScopedAlloc& Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; + IComputationPattern::TPtr Pattern; + }; + + struct TStreamWithYield : public NUdf::TBoxedValue { + TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index) + : Items(items) + , YieldPos(yieldPos) + , Index(index) + {} + + private: + TUnboxedValueVector Items; + ui32 YieldPos; + ui32 Index; + + ui32 GetTraverseCount() const override { + return 0; + } + + NUdf::TUnboxedValue Save() const override { + return NUdf::TUnboxedValue::Zero(); + } + + void Load(const NUdf::TStringRef& state) override { + Y_UNUSED(state); + } + + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final { + if (Index >= Items.size()) { + return NUdf::EFetchStatus::Finish; + } + if (Index == YieldPos) { + return NUdf::EFetchStatus::Yield; + } + result = Items[Index++]; + return NUdf::EFetchStatus::Ok; + } + }; + + THolder<IComputationGraph> BuildGraph(TSetup& setup, const std::vector<std::tuple<ui32, i64, ui32>> items, + ui32 yieldPos, ui32 startIndex, bool dataWatermarks) { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto structType = pgmBuilder.NewEmptyStructType(); + structType = pgmBuilder.NewStructType(structType, "key", + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + structType = pgmBuilder.NewStructType(structType, "time", + pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); + structType = pgmBuilder.NewStructType(structType, "sum", + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key"); + auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); + auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); + + auto inStreamType = pgmBuilder.NewStreamType(structType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType); + auto streamNode = inStream.Build(); + + ui64 hop = 10, interval = 30, delay = 20; + + auto pgmReturn = pgmBuilder.MultiHoppingCore( + TRuntimeNode(streamNode, false), + [&](TRuntimeNode item) { // keyExtractor + return pgmBuilder.Member(item, "key"); + }, + [&](TRuntimeNode item) { // timeExtractor + return pgmBuilder.Member(item, "time"); + }, + [&](TRuntimeNode item) { // init + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", pgmBuilder.Member(item, "sum")); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode item, TRuntimeNode state) { // update + auto add = pgmBuilder.AggrAdd( + pgmBuilder.Member(item, "sum"), + pgmBuilder.Member(state, "sum")); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", add); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state) { // save + return pgmBuilder.Member(state, "sum"); + }, + [&](TRuntimeNode savedState) { // load + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", savedState); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state1, TRuntimeNode state2) { // merge + auto add = pgmBuilder.AggrAdd( + pgmBuilder.Member(state1, "sum"), + pgmBuilder.Member(state2, "sum")); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("sum", add); + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish + Y_UNUSED(time); + std::vector<std::pair<std::string_view, TRuntimeNode>> members; + members.emplace_back("key", key); + members.emplace_back("sum", pgmBuilder.Member(state, "sum")); + return pgmBuilder.NewStruct(members); + }, + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay + pgmBuilder.NewDataLiteral<bool>(dataWatermarks) // dataWatermarks + ); + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + + TUnboxedValueVector streamItems; + for (size_t i = 0; i < items.size(); ++i) { + NUdf::TUnboxedValue* itemsPtr; + auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr); + itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(std::get<0>(items[i])); + itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(std::get<1>(items[i])); + itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(std::get<2>(items[i])); + streamItems.push_back(std::move(structValues)); + } + + auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex)); + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); + return graph; + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingSaveLoadTest) { + void TestWithSaveLoadImpl( + const std::vector<std::tuple<ui32, i64, ui32>> input, + const std::vector<std::tuple<ui32, ui32>> expected, + bool withTraverse, + bool dataWatermarks) + { + TScopedAlloc alloc; + + for (ui32 yieldPos = 0; yieldPos < input.size(); ++yieldPos) { + std::vector<std::tuple<ui32, ui32>> result; + + TSetup setup1(alloc); + auto graph1 = BuildGraph(setup1, input, yieldPos, 0, dataWatermarks); + auto root1 = graph1->GetValue(); + + NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root1.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>()); + } + } + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield); + + TString graphState; + if (withTraverse) { + SaveGraphState(&root1, 1, 0ULL, graphState); + } else { + graphState = graph1->SaveGraphState(); + } + + TSetup setup2(alloc); + auto graph2 = BuildGraph(setup2, input, -1, yieldPos, dataWatermarks); + NUdf::TUnboxedValue root2; + if (withTraverse) { + root2 = graph2->GetValue(); + LoadGraphState(&root2, 1, 0ULL, graphState); + } else { + graph2->LoadGraphState(graphState); + root2 = graph2->GetValue(); + } + + status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root2.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>()); + } + } + UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish); + + auto sortedExpected = expected; + std::sort(result.begin(), result.end()); + std::sort(sortedExpected.begin(), sortedExpected.end()); + UNIT_ASSERT_EQUAL(result, sortedExpected); + } + } + + const std::vector<std::tuple<ui32, i64, ui32>> input1 = { + // Group; Time; Value + {2, 1, 2}, + {1, 1, 2}, + {2, 2, 3}, + {1, 2, 3}, + {2, 15, 4}, + {1, 15, 4}, + {2, 23, 6}, + {1, 23, 6}, + {2, 24, 5}, + {1, 24, 5}, + {2, 25, 7}, + {1, 25, 7}, + {2, 40, 2}, + {1, 40, 2}, + {2, 47, 1}, + {1, 47, 1}, + {2, 51, 6}, + {1, 51, 6}, + {2, 59, 2}, + {1, 59, 2}, + {2, 85, 8}, + {1, 85, 8} + }; + + const std::vector<std::tuple<ui32, ui32>> expected = { + {1, 8}, {1, 8}, {1, 8}, {1, 8}, + {1, 11}, {1, 11}, {1, 21}, {1, 22}, + {1, 27}, + {2, 8}, {2, 8}, {2, 8}, {2, 8}, + {2, 11}, {2, 11}, {2, 21}, + {2, 22}, {2, 27}}; + + Y_UNIT_TEST(Test1) { + TestWithSaveLoadImpl(input1, expected, true, false); + } + + Y_UNIT_TEST(Test2) { + TestWithSaveLoadImpl(input1, expected, false, false); + } +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp index 1a2f47fe734..719ba1f6ea1 100644 --- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp @@ -7,400 +7,400 @@ #include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h> - -#include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { -namespace NMiniKQL { - -namespace { - struct TInputItem { - ui32 Key = 0; - i64 Time = 0; - ui32 Val = 0; - }; - - struct TOutputItem { - ui32 Key = 0; - ui32 Val = 0; - ui64 Time = 0; - - constexpr bool operator==(const TOutputItem& rhs) const - { - return this->Key == rhs.Key && this->Val == rhs.Val && this->Time == rhs.Time; - } - }; - - struct TOutputGroup { - TOutputGroup(std::initializer_list<TOutputItem> items) : Items(items) {} - - std::vector<TOutputItem> Items; - }; - - std::vector<TOutputItem> Ordered(std::vector<TOutputItem> vec) { - auto res = vec; - std::sort(res.begin(), res.end(), [](auto l, auto r) { - return std::make_tuple(l.Key, l.Val, l.Time) < std::make_tuple(r.Key, r.Val, r.Time); - }); - return res; - } - - IOutputStream &operator<<(IOutputStream &output, std::vector<TOutputItem> items) { - output << "["; - for (ui32 i = 0; i < items.size(); ++i) { - output << "(" << items.at(i).Key << ";" << items.at(i).Val << ";" << items.at(i).Time << ")"; - if (i != items.size() - 1) - output << ","; - } - output << "]"; - return output; - } - - TIntrusivePtr<IRandomProvider> CreateRandomProvider() { - return CreateDeterministicRandomProvider(1); - } - - TIntrusivePtr<ITimeProvider> CreateTimeProvider() { - return CreateDeterministicTimeProvider(10000000); - } - - TComputationNodeFactory GetAuxCallableFactory() { - return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { - if (callable.GetType()->GetName() == "MyStream") { - return new TExternalComputationNode(ctx.Mutables); - } - - return GetBuiltinFactory()(callable, ctx); - }; - } - - struct TSetup { - TSetup(TScopedAlloc& alloc) - : Alloc(alloc) - { - FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); - RandomProvider = CreateRandomProvider(); - TimeProvider = CreateTimeProvider(); - - Env.Reset(new TTypeEnvironment(Alloc)); - PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); - } - - THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { - Explorer.Walk(pgm.GetNode(), *Env); - TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(), - FunctionRegistry.Get(), - NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi); - Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); - TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); - return Pattern->Clone(compOpts); - } - - TIntrusivePtr<IFunctionRegistry> FunctionRegistry; - TIntrusivePtr<IRandomProvider> RandomProvider; - TIntrusivePtr<ITimeProvider> TimeProvider; - - TScopedAlloc& Alloc; - THolder<TTypeEnvironment> Env; - THolder<TProgramBuilder> PgmBuilder; - - TExploringNodeVisitor Explorer; - IComputationPattern::TPtr Pattern; - }; - - struct TStream : public NUdf::TBoxedValue { - TStream(const TUnboxedValueVector& items, std::function<void()> fetchCallback) - : Items(items) - , FetchCallback(fetchCallback) {} - - private: + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + struct TInputItem { + ui32 Key = 0; + i64 Time = 0; + ui32 Val = 0; + }; + + struct TOutputItem { + ui32 Key = 0; + ui32 Val = 0; + ui64 Time = 0; + + constexpr bool operator==(const TOutputItem& rhs) const + { + return this->Key == rhs.Key && this->Val == rhs.Val && this->Time == rhs.Time; + } + }; + + struct TOutputGroup { + TOutputGroup(std::initializer_list<TOutputItem> items) : Items(items) {} + + std::vector<TOutputItem> Items; + }; + + std::vector<TOutputItem> Ordered(std::vector<TOutputItem> vec) { + auto res = vec; + std::sort(res.begin(), res.end(), [](auto l, auto r) { + return std::make_tuple(l.Key, l.Val, l.Time) < std::make_tuple(r.Key, r.Val, r.Time); + }); + return res; + } + + IOutputStream &operator<<(IOutputStream &output, std::vector<TOutputItem> items) { + output << "["; + for (ui32 i = 0; i < items.size(); ++i) { + output << "(" << items.at(i).Key << ";" << items.at(i).Val << ";" << items.at(i).Time << ")"; + if (i != items.size() - 1) + output << ","; + } + output << "]"; + return output; + } + + TIntrusivePtr<IRandomProvider> CreateRandomProvider() { + return CreateDeterministicRandomProvider(1); + } + + TIntrusivePtr<ITimeProvider> CreateTimeProvider() { + return CreateDeterministicTimeProvider(10000000); + } + + TComputationNodeFactory GetAuxCallableFactory() { + return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (callable.GetType()->GetName() == "MyStream") { + return new TExternalComputationNode(ctx.Mutables); + } + + return GetBuiltinFactory()(callable, ctx); + }; + } + + struct TSetup { + TSetup(TScopedAlloc& alloc) + : Alloc(alloc) + { + FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry()); + RandomProvider = CreateRandomProvider(); + TimeProvider = CreateTimeProvider(); + + Env.Reset(new TTypeEnvironment(Alloc)); + PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry)); + } + + THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) { + Explorer.Walk(pgm.GetNode(), *Env); + TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(), + FunctionRegistry.Get(), + NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi); + Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts); + TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider); + return Pattern->Clone(compOpts); + } + + TIntrusivePtr<IFunctionRegistry> FunctionRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + + TScopedAlloc& Alloc; + THolder<TTypeEnvironment> Env; + THolder<TProgramBuilder> PgmBuilder; + + TExploringNodeVisitor Explorer; + IComputationPattern::TPtr Pattern; + }; + + struct TStream : public NUdf::TBoxedValue { + TStream(const TUnboxedValueVector& items, std::function<void()> fetchCallback) + : Items(items) + , FetchCallback(fetchCallback) {} + + private: TUnboxedValueVector Items; - ui32 Index; - std::function<void()> FetchCallback; - + ui32 Index; + std::function<void()> FetchCallback; + NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final { - FetchCallback(); - if (Index >= Items.size()) { - return NUdf::EFetchStatus::Finish; - } - result = Items[Index++]; - return NUdf::EFetchStatus::Ok; - } - }; - - THolder<IComputationGraph> BuildGraph( - TSetup& setup, - const std::vector<TInputItem> items, - std::function<void()> fetchCallback, - bool dataWatermarks, - ui64 hop = 10, - ui64 interval = 30, - ui64 delay = 20) - { - TProgramBuilder& pgmBuilder = *setup.PgmBuilder; - - auto structType = pgmBuilder.NewEmptyStructType(); + FetchCallback(); + if (Index >= Items.size()) { + return NUdf::EFetchStatus::Finish; + } + result = Items[Index++]; + return NUdf::EFetchStatus::Ok; + } + }; + + THolder<IComputationGraph> BuildGraph( + TSetup& setup, + const std::vector<TInputItem> items, + std::function<void()> fetchCallback, + bool dataWatermarks, + ui64 hop = 10, + ui64 interval = 30, + ui64 delay = 20) + { + TProgramBuilder& pgmBuilder = *setup.PgmBuilder; + + auto structType = pgmBuilder.NewEmptyStructType(); structType = pgmBuilder.NewStructType(structType, "key", - pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); structType = pgmBuilder.NewStructType(structType, "time", - pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); + pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id)); structType = pgmBuilder.NewStructType(structType, "sum", - pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); - auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key"); - auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); - auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); - - auto inStreamType = pgmBuilder.NewStreamType(structType); - - TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "MyStream", inStreamType); - auto streamNode = inStream.Build(); - - auto pgmReturn = pgmBuilder.MultiHoppingCore( - TRuntimeNode(streamNode, false), - [&](TRuntimeNode item) { // keyExtractor + pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id)); + auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key"); + auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time"); + auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum"); + + auto inStreamType = pgmBuilder.NewStreamType(structType); + + TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "MyStream", inStreamType); + auto streamNode = inStream.Build(); + + auto pgmReturn = pgmBuilder.MultiHoppingCore( + TRuntimeNode(streamNode, false), + [&](TRuntimeNode item) { // keyExtractor return pgmBuilder.Member(item, "key"); - }, - [&](TRuntimeNode item) { // timeExtractor + }, + [&](TRuntimeNode item) { // timeExtractor return pgmBuilder.Member(item, "time"); - }, - [&](TRuntimeNode item) { // init + }, + [&](TRuntimeNode item) { // init std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", pgmBuilder.Member(item, "sum")); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode item, TRuntimeNode state) { // update - auto add = pgmBuilder.AggrAdd( + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode item, TRuntimeNode state) { // update + auto add = pgmBuilder.AggrAdd( pgmBuilder.Member(item, "sum"), pgmBuilder.Member(state, "sum")); std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", add); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode state) { // save + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state) { // save return pgmBuilder.Member(state, "sum"); - }, - [&](TRuntimeNode savedState) { // load + }, + [&](TRuntimeNode savedState) { // load std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", savedState); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode state1, TRuntimeNode state2) { // merge - auto add = pgmBuilder.AggrAdd( + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode state1, TRuntimeNode state2) { // merge + auto add = pgmBuilder.AggrAdd( pgmBuilder.Member(state1, "sum"), pgmBuilder.Member(state2, "sum")); std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("sum", add); - return pgmBuilder.NewStruct(members); - }, - [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish + return pgmBuilder.NewStruct(members); + }, + [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish std::vector<std::pair<std::string_view, TRuntimeNode>> members; members.emplace_back("key", key); members.emplace_back("sum", pgmBuilder.Member(state, "sum")); - members.emplace_back("time", time); - return pgmBuilder.NewStruct(members); - }, - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval - pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay - pgmBuilder.NewDataLiteral<bool>(dataWatermarks) // dataWatermarks - ); - - auto graph = setup.BuildGraph(pgmReturn, {streamNode}); - + members.emplace_back("time", time); + return pgmBuilder.NewStruct(members); + }, + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval + pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay + pgmBuilder.NewDataLiteral<bool>(dataWatermarks) // dataWatermarks + ); + + auto graph = setup.BuildGraph(pgmReturn, {streamNode}); + TUnboxedValueVector streamItems; - for (size_t i = 0; i < items.size(); ++i) { - NUdf::TUnboxedValue* itemsPtr; - auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr); - itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(items.at(i).Key); - itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items.at(i).Time); - itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items.at(i).Val); - streamItems.push_back(std::move(structValues)); - } - - auto streamValue = NUdf::TUnboxedValuePod(new TStream(streamItems, fetchCallback)); - graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); - return graph; - } -} - -Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingTest) { - void TestImpl( - const std::vector<TInputItem> input, - const std::vector<TOutputGroup> expected, - bool dataWatermarks, - ui64 hop = 10, - ui64 interval = 30, - ui64 delay = 20) - { - TScopedAlloc alloc; - TSetup setup1(alloc); - - ui32 curGroupId = 0; - std::vector<TOutputItem> curResult; - - auto check = [&curResult, &curGroupId, &expected]() { - auto expectedItems = Ordered(expected.at(curGroupId).Items); - curResult = Ordered(curResult); - UNIT_ASSERT_EQUAL_C(curResult, expectedItems, "curGroup: " << curGroupId << " actual: " << curResult << " expected: " << expectedItems); - curGroupId++; - curResult.clear(); - }; - - auto graph1 = BuildGraph(setup1, input, check, dataWatermarks, hop, interval, delay); - - auto root1 = graph1->GetValue(); - - NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; - while (status == NUdf::EFetchStatus::Ok) { - NUdf::TUnboxedValue val; - status = root1.Fetch(val); - if (status == NUdf::EFetchStatus::Ok) { - curResult.emplace_back(TOutputItem{val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>(), val.GetElement(2).Get<ui64>()}); + for (size_t i = 0; i < items.size(); ++i) { + NUdf::TUnboxedValue* itemsPtr; + auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr); + itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(items.at(i).Key); + itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items.at(i).Time); + itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items.at(i).Val); + streamItems.push_back(std::move(structValues)); + } + + auto streamValue = NUdf::TUnboxedValuePod(new TStream(streamItems, fetchCallback)); + graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue)); + return graph; + } +} + +Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingTest) { + void TestImpl( + const std::vector<TInputItem> input, + const std::vector<TOutputGroup> expected, + bool dataWatermarks, + ui64 hop = 10, + ui64 interval = 30, + ui64 delay = 20) + { + TScopedAlloc alloc; + TSetup setup1(alloc); + + ui32 curGroupId = 0; + std::vector<TOutputItem> curResult; + + auto check = [&curResult, &curGroupId, &expected]() { + auto expectedItems = Ordered(expected.at(curGroupId).Items); + curResult = Ordered(curResult); + UNIT_ASSERT_EQUAL_C(curResult, expectedItems, "curGroup: " << curGroupId << " actual: " << curResult << " expected: " << expectedItems); + curGroupId++; + curResult.clear(); + }; + + auto graph1 = BuildGraph(setup1, input, check, dataWatermarks, hop, interval, delay); + + auto root1 = graph1->GetValue(); + + NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok; + while (status == NUdf::EFetchStatus::Ok) { + NUdf::TUnboxedValue val; + status = root1.Fetch(val); + if (status == NUdf::EFetchStatus::Ok) { + curResult.emplace_back(TOutputItem{val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>(), val.GetElement(2).Get<ui64>()}); } - } - - check(); - // TODO: some problem with parallel run - //UNIT_ASSERT_EQUAL_C(curGroupId, expected.size(), "1: " << curGroupId << " 2: " << expected.size()); - } - - Y_UNIT_TEST(TestDataWatermarks) { - const std::vector<TInputItem> input = { - // Group; Time; Value - {1, 101, 2}, - {2, 101, 2}, - {1, 111, 3}, - {2, 140, 5}, - {2, 160, 1} - }; - const std::vector<TOutputGroup> expected = { - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({{1, 2, 110}, {1, 5, 120}, {2, 2, 110}, {2, 2, 120}}), - TOutputGroup({{2, 2, 130}, {1, 5, 130}, {1, 3, 140}}), - TOutputGroup({{2, 5, 150}, {2, 5, 160}, {2, 6, 170}, {2, 1, 180}, {2, 1, 190}}), - }; - TestImpl(input, expected, true); - } - - Y_UNIT_TEST(TestValidness1) { - const std::vector<TInputItem> input1 = { - // Group; Time; Value - {1, 101, 2}, - {2, 101, 2}, - {1, 111, 3}, - {2, 140, 5}, - {2, 160, 1} - }; - - const std::vector<TOutputGroup> expected = { - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({{2, 2, 110}, {2, 2, 120}}), - TOutputGroup({{2, 2, 130}}), - TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}, {2, 5, 150}, - {2, 5, 160}, {2, 6, 170}, {2, 1, 190}, {2, 1, 180}}), - }; - TestImpl(input1, expected, false); - } - - Y_UNIT_TEST(TestValidness2) { - const std::vector<TInputItem> input = { - // Group; Time; Value - {2, 101, 2}, {1, 101, 2}, {2, 102, 3}, {1, 102, 3}, {2, 115, 4}, - {1, 115, 4}, {2, 123, 6}, {1, 123, 6}, {2, 124, 5}, {1, 124, 5}, - {2, 125, 7}, {1, 125, 7}, {2, 140, 2}, {1, 140, 2}, {2, 147, 1}, - {1, 147, 1}, {2, 151, 6}, {1, 151, 6}, {2, 159, 2}, {1, 159, 2}, - {2, 185, 8}, {1, 185, 8} - }; - const std::vector<TOutputGroup> expected = { - TOutputGroup({}), - TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), - TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), - TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), - TOutputGroup({{1, 5, 110}, {1, 9, 120}, {2, 5, 110}, {2, 9, 120}}), - TOutputGroup({}), - TOutputGroup({}), TOutputGroup({}), - TOutputGroup({{2, 27, 130}, {1, 27, 130}}), - TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), - TOutputGroup({{2, 22, 140}, {2, 21, 150}, {2, 11, 160}, {1, 22, 140}, {1, 21, 150}, {1, 11, 160}}), - TOutputGroup({}), - TOutputGroup({{1, 11, 170}, {1, 8, 180}, {1, 8, 190}, {1, 8, 200}, {1, 8, 210}, {2, 11, 170}, - {2, 8, 180}, {2, 8, 190}, {2, 8, 200}, {2, 8, 210}}), - }; - - TestImpl(input, expected, true); - } - - Y_UNIT_TEST(TestValidness3) { - const std::vector<TInputItem> input = { - // Group; Time; Value - {1, 105, 1}, {1, 107, 4}, {2, 106, 3}, {1, 111, 7}, {1, 117, 3}, - {2, 110, 2}, {1, 108, 9}, {1, 121, 4}, {2, 107, 2}, {2, 141, 5}, - {1, 141, 10} - }; - const std::vector<TOutputGroup> expected = { - TOutputGroup({}), - TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), - TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), - TOutputGroup({{1, 14, 110}, {2, 3, 110}}), - TOutputGroup({}), - TOutputGroup({{2, 7, 115}, {2, 2, 120}, {1, 21, 115}, {1, 10, 120}, {1, 7, 125}, {1, 4, 130}}), - TOutputGroup({}), - TOutputGroup({{1, 10, 145}, {1, 10, 150}, {2, 5, 145}, {2, 5, 150}}) - }; - - TestImpl(input, expected, true, 5, 10, 10); + } + + check(); + // TODO: some problem with parallel run + //UNIT_ASSERT_EQUAL_C(curGroupId, expected.size(), "1: " << curGroupId << " 2: " << expected.size()); + } + + Y_UNIT_TEST(TestDataWatermarks) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 2}, + {2, 101, 2}, + {1, 111, 3}, + {2, 140, 5}, + {2, 160, 1} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 2, 110}, {1, 5, 120}, {2, 2, 110}, {2, 2, 120}}), + TOutputGroup({{2, 2, 130}, {1, 5, 130}, {1, 3, 140}}), + TOutputGroup({{2, 5, 150}, {2, 5, 160}, {2, 6, 170}, {2, 1, 180}, {2, 1, 190}}), + }; + TestImpl(input, expected, true); + } + + Y_UNIT_TEST(TestValidness1) { + const std::vector<TInputItem> input1 = { + // Group; Time; Value + {1, 101, 2}, + {2, 101, 2}, + {1, 111, 3}, + {2, 140, 5}, + {2, 160, 1} + }; + + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{2, 2, 110}, {2, 2, 120}}), + TOutputGroup({{2, 2, 130}}), + TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}, {2, 5, 150}, + {2, 5, 160}, {2, 6, 170}, {2, 1, 190}, {2, 1, 180}}), + }; + TestImpl(input1, expected, false); + } + + Y_UNIT_TEST(TestValidness2) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {2, 101, 2}, {1, 101, 2}, {2, 102, 3}, {1, 102, 3}, {2, 115, 4}, + {1, 115, 4}, {2, 123, 6}, {1, 123, 6}, {2, 124, 5}, {1, 124, 5}, + {2, 125, 7}, {1, 125, 7}, {2, 140, 2}, {1, 140, 2}, {2, 147, 1}, + {1, 147, 1}, {2, 151, 6}, {1, 151, 6}, {2, 159, 2}, {1, 159, 2}, + {2, 185, 8}, {1, 185, 8} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{1, 5, 110}, {1, 9, 120}, {2, 5, 110}, {2, 9, 120}}), + TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{2, 27, 130}, {1, 27, 130}}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{2, 22, 140}, {2, 21, 150}, {2, 11, 160}, {1, 22, 140}, {1, 21, 150}, {1, 11, 160}}), + TOutputGroup({}), + TOutputGroup({{1, 11, 170}, {1, 8, 180}, {1, 8, 190}, {1, 8, 200}, {1, 8, 210}, {2, 11, 170}, + {2, 8, 180}, {2, 8, 190}, {2, 8, 200}, {2, 8, 210}}), + }; + + TestImpl(input, expected, true); + } + + Y_UNIT_TEST(TestValidness3) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 105, 1}, {1, 107, 4}, {2, 106, 3}, {1, 111, 7}, {1, 117, 3}, + {2, 110, 2}, {1, 108, 9}, {1, 121, 4}, {2, 107, 2}, {2, 141, 5}, + {1, 141, 10} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{1, 14, 110}, {2, 3, 110}}), + TOutputGroup({}), + TOutputGroup({{2, 7, 115}, {2, 2, 120}, {1, 21, 115}, {1, 10, 120}, {1, 7, 125}, {1, 4, 130}}), + TOutputGroup({}), + TOutputGroup({{1, 10, 145}, {1, 10, 150}, {2, 5, 145}, {2, 5, 150}}) + }; + + TestImpl(input, expected, true, 5, 10, 10); } - Y_UNIT_TEST(TestDelay) { - const std::vector<TInputItem> input = { - // Group; Time; Value - {1, 101, 3}, {1, 111, 5}, {1, 120, 7}, {1, 80, 9}, {1, 79, 11} - }; - const std::vector<TOutputGroup> expected = { - TOutputGroup({}), - TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), - TOutputGroup({}), TOutputGroup({}), - TOutputGroup({{1, 12, 110}, {1, 8, 120}, {1, 15, 130}, {1, 12, 140}, {1, 7, 150}}) - }; - - TestImpl(input, expected, false); + Y_UNIT_TEST(TestDelay) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 3}, {1, 111, 5}, {1, 120, 7}, {1, 80, 9}, {1, 79, 11} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), + TOutputGroup({}), TOutputGroup({}), + TOutputGroup({{1, 12, 110}, {1, 8, 120}, {1, 15, 130}, {1, 12, 140}, {1, 7, 150}}) + }; + + TestImpl(input, expected, false); + } + + Y_UNIT_TEST(TestWindowsBeforeFirstElement) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 101, 2}, {1, 111, 3} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}}) + }; + + TestImpl(input, expected, false); } - - Y_UNIT_TEST(TestWindowsBeforeFirstElement) { - const std::vector<TInputItem> input = { - // Group; Time; Value - {1, 101, 2}, {1, 111, 3} - }; - const std::vector<TOutputGroup> expected = { - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}}) - }; - - TestImpl(input, expected, false); - } - - Y_UNIT_TEST(TestSubzeroValues) { - const std::vector<TInputItem> input = { - // Group; Time; Value - {1, 1, 2} - }; - const std::vector<TOutputGroup> expected = { - TOutputGroup({}), - TOutputGroup({}), - TOutputGroup({{1, 2, 30}}), - }; - - TestImpl(input, expected, false); - } -} - -} // namespace NMiniKQL -} // namespace NKikimr + + Y_UNIT_TEST(TestSubzeroValues) { + const std::vector<TInputItem> input = { + // Group; Time; Value + {1, 1, 2} + }; + const std::vector<TOutputGroup> expected = { + TOutputGroup({}), + TOutputGroup({}), + TOutputGroup({{1, 2, 30}}), + }; + + TestImpl(input, expected, false); + } +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/ut/ya.make b/ydb/library/yql/minikql/comp_nodes/ut/ya.make index 38243bc1d37..dffda1318f3 100644 --- a/ydb/library/yql/minikql/comp_nodes/ut/ya.make +++ b/ydb/library/yql/minikql/comp_nodes/ut/ya.make @@ -27,8 +27,8 @@ SRCS( mkql_chopper_ut.cpp mkql_filters_ut.cpp mkql_flatmap_ut.cpp - mkql_multihopping_saveload_ut.cpp - mkql_multihopping_ut.cpp + mkql_multihopping_saveload_ut.cpp + mkql_multihopping_ut.cpp mkql_multimap_ut.cpp mkql_fold_ut.cpp mkql_heap_ut.cpp diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make index b4b5ee777bb..55838c40123 100644 --- a/ydb/library/yql/minikql/comp_nodes/ya.make +++ b/ydb/library/yql/minikql/comp_nodes/ya.make @@ -112,8 +112,8 @@ SRCS( mkql_map.h mkql_map_join.cpp mkql_map_join.h - mkql_multihopping.cpp - mkql_multihopping.h + mkql_multihopping.cpp + mkql_multihopping.h mkql_multimap.cpp mkql_multimap.h mkql_next_value.cpp diff --git a/ydb/library/yql/minikql/mkql_alloc.h b/ydb/library/yql/minikql/mkql_alloc.h index ee254585394..5e864cdc4c8 100644 --- a/ydb/library/yql/minikql/mkql_alloc.h +++ b/ydb/library/yql/minikql/mkql_alloc.h @@ -120,7 +120,7 @@ public: void InvalidateMemInfo() { MyState_.InvalidateMemInfo(); } bool IsAttached() const { return AttachedCount_ > 0; } - + private: TAllocState MyState_; size_t AttachedCount_ = 0; diff --git a/ydb/library/yql/minikql/mkql_node.h b/ydb/library/yql/minikql/mkql_node.h index e7f9c9ec60d..ee74455ea69 100644 --- a/ydb/library/yql/minikql/mkql_node.h +++ b/ydb/library/yql/minikql/mkql_node.h @@ -462,8 +462,8 @@ public: return Guard(Alloc); } - TScopedAlloc& GetAllocator() const { return Alloc; } - + TScopedAlloc& GetAllocator() const { return Alloc; } + const NUdf::TStringValue& NewString(ui32 size) const { Strings.emplace(size); return Strings.top(); diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index de7b56c0de9..47ae6e4c8fc 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -4660,89 +4660,89 @@ TRuntimeNode TProgramBuilder::HoppingCore(TRuntimeNode list, return TRuntimeNode(callableBuilder.Build(), false); } -TRuntimeNode TProgramBuilder::MultiHoppingCore(TRuntimeNode list, - const TUnaryLambda& keyExtractor, - const TUnaryLambda& timeExtractor, - const TUnaryLambda& init, - const TBinaryLambda& update, - const TUnaryLambda& save, - const TUnaryLambda& load, - const TBinaryLambda& merge, - const TTernaryLambda& finish, - TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay, TRuntimeNode dataWatermarks) -{ - if constexpr (RuntimeVersion < 22U) { - THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; - } - - auto streamType = AS_TYPE(TStreamType, list); - auto itemType = AS_TYPE(TStructType, streamType->GetItemType()); - auto timestampType = TOptionalType::Create(TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, Env), Env); - - TRuntimeNode itemArg = Arg(itemType); - - auto keyExtract = keyExtractor(itemArg); - auto keyType = keyExtract.GetStaticType(); - TRuntimeNode keyArg = Arg(keyType); - - auto outTime = timeExtractor(itemArg); - auto outStateInit = init(itemArg); - - auto stateType = outStateInit.GetStaticType(); - TRuntimeNode stateArg = Arg(stateType); - - auto outStateUpdate = update(itemArg, stateArg); - - auto hasSaveLoad = (bool)save; - TRuntimeNode saveArg, outSave, loadArg, outLoad; - if (hasSaveLoad) { - saveArg = Arg(stateType); - outSave = save(saveArg); - - loadArg = Arg(outSave.GetStaticType()); - outLoad = load(loadArg); - - MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*stateType), "Loaded type is changed by the load handler"); - } else { - saveArg = outSave = loadArg = outLoad = NewVoid(); - } - - TRuntimeNode state2Arg = Arg(stateType); - TRuntimeNode timeArg = Arg(timestampType); - - auto outStateMerge = merge(stateArg, state2Arg); - auto outItemFinish = finish(keyArg, stateArg, timeArg); - - auto finishType = outItemFinish.GetStaticType(); - MKQL_ENSURE(finishType->IsStruct(), "Expected struct type as finish lambda output"); - - auto resultType = TStreamType::Create(outItemFinish.GetStaticType(), Env); - - TCallableBuilder callableBuilder(Env, __func__, resultType); - callableBuilder.Add(list); - callableBuilder.Add(itemArg); - callableBuilder.Add(keyArg); - callableBuilder.Add(stateArg); - callableBuilder.Add(state2Arg); - callableBuilder.Add(timeArg); - callableBuilder.Add(saveArg); - callableBuilder.Add(loadArg); - callableBuilder.Add(keyExtract); - callableBuilder.Add(outTime); - callableBuilder.Add(outStateInit); - callableBuilder.Add(outStateUpdate); - callableBuilder.Add(outSave); - callableBuilder.Add(outLoad); - callableBuilder.Add(outStateMerge); - callableBuilder.Add(outItemFinish); - callableBuilder.Add(hop); - callableBuilder.Add(interval); - callableBuilder.Add(delay); - callableBuilder.Add(dataWatermarks); - - return TRuntimeNode(callableBuilder.Build(), false); -} - +TRuntimeNode TProgramBuilder::MultiHoppingCore(TRuntimeNode list, + const TUnaryLambda& keyExtractor, + const TUnaryLambda& timeExtractor, + const TUnaryLambda& init, + const TBinaryLambda& update, + const TUnaryLambda& save, + const TUnaryLambda& load, + const TBinaryLambda& merge, + const TTernaryLambda& finish, + TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay, TRuntimeNode dataWatermarks) +{ + if constexpr (RuntimeVersion < 22U) { + THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; + } + + auto streamType = AS_TYPE(TStreamType, list); + auto itemType = AS_TYPE(TStructType, streamType->GetItemType()); + auto timestampType = TOptionalType::Create(TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, Env), Env); + + TRuntimeNode itemArg = Arg(itemType); + + auto keyExtract = keyExtractor(itemArg); + auto keyType = keyExtract.GetStaticType(); + TRuntimeNode keyArg = Arg(keyType); + + auto outTime = timeExtractor(itemArg); + auto outStateInit = init(itemArg); + + auto stateType = outStateInit.GetStaticType(); + TRuntimeNode stateArg = Arg(stateType); + + auto outStateUpdate = update(itemArg, stateArg); + + auto hasSaveLoad = (bool)save; + TRuntimeNode saveArg, outSave, loadArg, outLoad; + if (hasSaveLoad) { + saveArg = Arg(stateType); + outSave = save(saveArg); + + loadArg = Arg(outSave.GetStaticType()); + outLoad = load(loadArg); + + MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*stateType), "Loaded type is changed by the load handler"); + } else { + saveArg = outSave = loadArg = outLoad = NewVoid(); + } + + TRuntimeNode state2Arg = Arg(stateType); + TRuntimeNode timeArg = Arg(timestampType); + + auto outStateMerge = merge(stateArg, state2Arg); + auto outItemFinish = finish(keyArg, stateArg, timeArg); + + auto finishType = outItemFinish.GetStaticType(); + MKQL_ENSURE(finishType->IsStruct(), "Expected struct type as finish lambda output"); + + auto resultType = TStreamType::Create(outItemFinish.GetStaticType(), Env); + + TCallableBuilder callableBuilder(Env, __func__, resultType); + callableBuilder.Add(list); + callableBuilder.Add(itemArg); + callableBuilder.Add(keyArg); + callableBuilder.Add(stateArg); + callableBuilder.Add(state2Arg); + callableBuilder.Add(timeArg); + callableBuilder.Add(saveArg); + callableBuilder.Add(loadArg); + callableBuilder.Add(keyExtract); + callableBuilder.Add(outTime); + callableBuilder.Add(outStateInit); + callableBuilder.Add(outStateUpdate); + callableBuilder.Add(outSave); + callableBuilder.Add(outLoad); + callableBuilder.Add(outStateMerge); + callableBuilder.Add(outItemFinish); + callableBuilder.Add(hop); + callableBuilder.Add(interval); + callableBuilder.Add(delay); + callableBuilder.Add(dataWatermarks); + + return TRuntimeNode(callableBuilder.Build(), false); +} + TRuntimeNode TProgramBuilder::Default(TType* type) { bool isOptional; const auto targetType = UnpackOptionalData(type, isOptional); diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index fcaf39d5e5e..10f1ad7ccf1 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -425,16 +425,16 @@ public: const TBinaryLambda& merge, const TBinaryLambda& finish, TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay); - TRuntimeNode MultiHoppingCore(TRuntimeNode list, - const TUnaryLambda& keyExtractor, - const TUnaryLambda& timeExtractor, - const TUnaryLambda& init, - const TBinaryLambda& update, - const TUnaryLambda& save, - const TUnaryLambda& load, - const TBinaryLambda& merge, - const TTernaryLambda& finish, - TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay, TRuntimeNode dataWatermarks); + TRuntimeNode MultiHoppingCore(TRuntimeNode list, + const TUnaryLambda& keyExtractor, + const TUnaryLambda& timeExtractor, + const TUnaryLambda& init, + const TBinaryLambda& update, + const TUnaryLambda& save, + const TUnaryLambda& load, + const TBinaryLambda& merge, + const TTernaryLambda& finish, + TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay, TRuntimeNode dataWatermarks); TRuntimeNode Chopper(TRuntimeNode flow, const TUnaryLambda& keyExtractor, const TBinaryLambda& groupSwitch, const TBinaryLambda& groupHandler); diff --git a/ydb/library/yql/minikql/ut/ya.make b/ydb/library/yql/minikql/ut/ya.make index 74000c251c9..098a77d0c60 100644 --- a/ydb/library/yql/minikql/ut/ya.make +++ b/ydb/library/yql/minikql/ut/ya.make @@ -28,7 +28,7 @@ SRCS( mkql_type_ops_ut.cpp mkql_string_util_ut.cpp pack_num_ut.cpp - watermark_tracker_ut.cpp + watermark_tracker_ut.cpp ) PEERDIR( diff --git a/ydb/library/yql/minikql/watermark_tracker.cpp b/ydb/library/yql/minikql/watermark_tracker.cpp index b359c513b97..2e08bb08e76 100644 --- a/ydb/library/yql/minikql/watermark_tracker.cpp +++ b/ydb/library/yql/minikql/watermark_tracker.cpp @@ -1,39 +1,39 @@ -#include "watermark_tracker.h" - -#include<util/system/yassert.h> - -namespace NKikimr { -namespace NMiniKQL { - -TWatermarkTracker::TWatermarkTracker( - ui64 delay, - ui64 granularity) - : Delay(delay) - , Granularity(granularity) -{ - Y_VERIFY(granularity > 0); -} - -std::optional<ui64> TWatermarkTracker::HandleNextEventTime(ui64 ts) { - if (Y_UNLIKELY(ts >= NextEventWithWatermark)) { - NextEventWithWatermark = CalcNextEventWithWatermark(ts); - return CalcLastWatermark(); - } - - return std::nullopt; -} - -ui64 TWatermarkTracker::CalcNextEventWithWatermark(ui64 ts) { - return ts + Granularity - (ts - Delay) % Granularity; -} - -std::optional<ui64> TWatermarkTracker::CalcLastWatermark() { - if (Y_UNLIKELY(Delay + Granularity > NextEventWithWatermark)) { - // Protect from negative values - return std::nullopt; - } - return NextEventWithWatermark - Delay - Granularity; -} - -} // NMiniKQL -} // NKikimr +#include "watermark_tracker.h" + +#include<util/system/yassert.h> + +namespace NKikimr { +namespace NMiniKQL { + +TWatermarkTracker::TWatermarkTracker( + ui64 delay, + ui64 granularity) + : Delay(delay) + , Granularity(granularity) +{ + Y_VERIFY(granularity > 0); +} + +std::optional<ui64> TWatermarkTracker::HandleNextEventTime(ui64 ts) { + if (Y_UNLIKELY(ts >= NextEventWithWatermark)) { + NextEventWithWatermark = CalcNextEventWithWatermark(ts); + return CalcLastWatermark(); + } + + return std::nullopt; +} + +ui64 TWatermarkTracker::CalcNextEventWithWatermark(ui64 ts) { + return ts + Granularity - (ts - Delay) % Granularity; +} + +std::optional<ui64> TWatermarkTracker::CalcLastWatermark() { + if (Y_UNLIKELY(Delay + Granularity > NextEventWithWatermark)) { + // Protect from negative values + return std::nullopt; + } + return NextEventWithWatermark - Delay - Granularity; +} + +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/watermark_tracker.h b/ydb/library/yql/minikql/watermark_tracker.h index 7926c4b583d..aa06f614f4e 100644 --- a/ydb/library/yql/minikql/watermark_tracker.h +++ b/ydb/library/yql/minikql/watermark_tracker.h @@ -1,25 +1,25 @@ -#pragma once - -#include <optional> -#include <util/system/types.h> - -namespace NKikimr { -namespace NMiniKQL { - -class TWatermarkTracker { -public: - TWatermarkTracker(ui64 lag, ui64 granularity); - std::optional<ui64> HandleNextEventTime(ui64 ts); - -private: - ui64 CalcNextEventWithWatermark(ui64 ts); - std::optional<ui64> CalcLastWatermark(); - -private: - ui64 NextEventWithWatermark = 0; - const ui64 Delay; - const ui64 Granularity; -}; - -} // NMiniKQL -} // NKikimr +#pragma once + +#include <optional> +#include <util/system/types.h> + +namespace NKikimr { +namespace NMiniKQL { + +class TWatermarkTracker { +public: + TWatermarkTracker(ui64 lag, ui64 granularity); + std::optional<ui64> HandleNextEventTime(ui64 ts); + +private: + ui64 CalcNextEventWithWatermark(ui64 ts); + std::optional<ui64> CalcLastWatermark(); + +private: + ui64 NextEventWithWatermark = 0; + const ui64 Delay; + const ui64 Granularity; +}; + +} // NMiniKQL +} // NKikimr diff --git a/ydb/library/yql/minikql/watermark_tracker_ut.cpp b/ydb/library/yql/minikql/watermark_tracker_ut.cpp index 33cc3492458..ab2864caa21 100644 --- a/ydb/library/yql/minikql/watermark_tracker_ut.cpp +++ b/ydb/library/yql/minikql/watermark_tracker_ut.cpp @@ -1,26 +1,26 @@ -#include "watermark_tracker.h" - -#include <util/system/types.h> -#include <library/cpp/testing/unittest/registar.h> - -namespace NKikimr { -namespace NMiniKQL { - -Y_UNIT_TEST_SUITE(TWatermarkTrackerTest) { - Y_UNIT_TEST(WhenNotGranularShouldNotMoveWatermark) { - TWatermarkTracker tracker(15, 10); - UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(25), 10); - UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(26), std::nullopt); - UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(30), std::nullopt); - UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(34), std::nullopt); - } - - Y_UNIT_TEST(WhenGranularShouldMoveWatermark) { - TWatermarkTracker tracker(15, 10); - UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(35), 20); - UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(45), 30); - } -} - -} // NMiniKQL -} // NKikimr
\ No newline at end of file +#include "watermark_tracker.h" + +#include <util/system/types.h> +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr { +namespace NMiniKQL { + +Y_UNIT_TEST_SUITE(TWatermarkTrackerTest) { + Y_UNIT_TEST(WhenNotGranularShouldNotMoveWatermark) { + TWatermarkTracker tracker(15, 10); + UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(25), 10); + UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(26), std::nullopt); + UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(30), std::nullopt); + UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(34), std::nullopt); + } + + Y_UNIT_TEST(WhenGranularShouldMoveWatermark) { + TWatermarkTracker tracker(15, 10); + UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(35), 20); + UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(45), 30); + } +} + +} // NMiniKQL +} // NKikimr
\ No newline at end of file diff --git a/ydb/library/yql/minikql/ya.make b/ydb/library/yql/minikql/ya.make index 2df85e818c0..c4b608128d8 100644 --- a/ydb/library/yql/minikql/ya.make +++ b/ydb/library/yql/minikql/ya.make @@ -48,8 +48,8 @@ SRCS( mkql_unboxed_value_stream.h primes.cpp primes.h - watermark_tracker.cpp - watermark_tracker.h + watermark_tracker.cpp + watermark_tracker.h ) PEERDIR( diff --git a/ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp b/ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp index 94adf7ba30a..88aea459c33 100644 --- a/ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp +++ b/ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp @@ -2,13 +2,13 @@ #include "yql_ch_read_actor.h" #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> - + namespace NYql::NDq { void RegisterClickHouseReadActorFactory(TDqSourceFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IHTTPGateway::TPtr gateway) { factory.Register<NCH::TSource>("ClickHouseSource", - [credentialsFactory, gateway](NCH::TSource&& settings, IDqSourceActorFactory::TArguments&& args) { - return CreateClickHouseReadActor(gateway, std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, credentialsFactory); + [credentialsFactory, gateway](NCH::TSource&& settings, IDqSourceActorFactory::TArguments&& args) { + return CreateClickHouseReadActor(gateway, std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, credentialsFactory); }); } diff --git a/ydb/library/yql/providers/common/mkql/parser.cpp b/ydb/library/yql/providers/common/mkql/parser.cpp index 1c0fda5860d..bfdc26f5f85 100644 --- a/ydb/library/yql/providers/common/mkql/parser.cpp +++ b/ydb/library/yql/providers/common/mkql/parser.cpp @@ -1,105 +1,105 @@ -#include "parser.h" -#include "util/generic/maybe.h" - -#include <functional> -#include <string_view> +#include "parser.h" +#include "util/generic/maybe.h" + +#include <functional> +#include <string_view> #include <ydb/library/yql/minikql/defs.h> - -#include <library/cpp/json/json_writer.h> - -namespace NYql { - -using namespace NKikimr::NMiniKQL; -using namespace NNodes; - -namespace { - -const TExprNode& GetFormat(const TExprNode& settings) { - for (auto i = 0U; i < settings.ChildrenSize(); ++i) { - const auto& child = *settings.Child(i); - if (child.Head().IsAtom("format")) - return child.Tail(); - } - THROW yexception() << "Unknown format."; -} - -std::array<TString, 2U> GetSettings(const TExprNode& settings) { - for (auto i = 0U; i < settings.ChildrenSize(); ++i) { - const auto& child = *settings.Child(i); - if (child.Head().IsAtom("settings")) { - if (child.Tail().IsList()) { - TString json, compression; - TStringOutput stream(json); - NJson::TJsonWriter writer(&stream, NJson::TJsonWriterConfig()); - writer.OpenMap(); - child.Tail().ForEachChild([&writer, &compression](const TExprNode& pair) { - if (pair.Head().IsAtom("compression") && pair.Tail().IsCallable({"String", "Utf8"})) - if (const auto& comp = pair.Tail().Head().Content(); !comp.empty() && std::isupper(comp.front())) - compression = comp; - else { - writer.WriteKey(pair.Head().Content()); - writer.Write(comp); - } - else if (pair.Head().IsAtom() && pair.Tail().IsCallable({"Bool", "Float", "Double", "Int8", "Uint8", "Int16", "Uint16", "Int32", "Uint32", "Int64", "Uint64", "String", "Utf8"})) { - writer.WriteKey(pair.Head().Content()); - if (const auto& type = pair.Tail().Content(); "Bool" == type) - writer.Write(FromString<bool>(pair.Tail().Head().Content())); - else if ("Float" == type) - writer.Write(FromString<float>(pair.Tail().Head().Content())); - else if ("Double" == type) - writer.Write(FromString<double>(pair.Tail().Head().Content())); - else if ("Int8" == type) - writer.Write(FromString<i8>(pair.Tail().Head().Content())); - else if ("Uint8" == type) - writer.Write(FromString<ui8>(pair.Tail().Head().Content())); - else if ("Int16" == type) - writer.Write(FromString<i16>(pair.Tail().Head().Content())); - else if ("Uint16" == type) - writer.Write(FromString<ui16>(pair.Tail().Head().Content())); - else if ("Int32" == type) - writer.Write(FromString<i32>(pair.Tail().Head().Content())); - else if ("Uint32" == type) - writer.Write(FromString<ui32>(pair.Tail().Head().Content())); - else if ("Int64" == type) - writer.Write(FromString<i64>(pair.Tail().Head().Content())); - else if ("Uint64" == type) - writer.Write(FromString<ui64>(pair.Tail().Head().Content())); - else - writer.Write(pair.Tail().Head().Content()); - - } - }); - writer.CloseMap(); - writer.Flush(); - if (json == "{}") - json.clear(); - return {std::move(json), std::move(compression)}; - } - } - } - return {TString(), TString()}; -} -} - -TRuntimeNode BuildParseCall( - TRuntimeNode input, - const std::string_view& format, - const std::string_view& compression, - TType* inputItemType, - TType* outputItemType, - NCommon::TMkqlBuildContext& ctx) -{ - if (!compression.empty()) { - input = ctx.ProgramBuilder.Map(input, [&ctx, &compression](TRuntimeNode item) { - return ctx.ProgramBuilder.Apply(ctx.ProgramBuilder.Udf(std::string("Decompress.") += compression), {item}); - }); - } - - const auto structType = static_cast<const TStructType*>(outputItemType); - if (format == "raw") { - MKQL_ENSURE(1U == structType->GetMembersCount(), "Expected single column."); - bool isOptional; - const auto schemeType = UnpackOptionalData(structType->GetMemberType(0U), isOptional)->GetSchemeType(); + +#include <library/cpp/json/json_writer.h> + +namespace NYql { + +using namespace NKikimr::NMiniKQL; +using namespace NNodes; + +namespace { + +const TExprNode& GetFormat(const TExprNode& settings) { + for (auto i = 0U; i < settings.ChildrenSize(); ++i) { + const auto& child = *settings.Child(i); + if (child.Head().IsAtom("format")) + return child.Tail(); + } + THROW yexception() << "Unknown format."; +} + +std::array<TString, 2U> GetSettings(const TExprNode& settings) { + for (auto i = 0U; i < settings.ChildrenSize(); ++i) { + const auto& child = *settings.Child(i); + if (child.Head().IsAtom("settings")) { + if (child.Tail().IsList()) { + TString json, compression; + TStringOutput stream(json); + NJson::TJsonWriter writer(&stream, NJson::TJsonWriterConfig()); + writer.OpenMap(); + child.Tail().ForEachChild([&writer, &compression](const TExprNode& pair) { + if (pair.Head().IsAtom("compression") && pair.Tail().IsCallable({"String", "Utf8"})) + if (const auto& comp = pair.Tail().Head().Content(); !comp.empty() && std::isupper(comp.front())) + compression = comp; + else { + writer.WriteKey(pair.Head().Content()); + writer.Write(comp); + } + else if (pair.Head().IsAtom() && pair.Tail().IsCallable({"Bool", "Float", "Double", "Int8", "Uint8", "Int16", "Uint16", "Int32", "Uint32", "Int64", "Uint64", "String", "Utf8"})) { + writer.WriteKey(pair.Head().Content()); + if (const auto& type = pair.Tail().Content(); "Bool" == type) + writer.Write(FromString<bool>(pair.Tail().Head().Content())); + else if ("Float" == type) + writer.Write(FromString<float>(pair.Tail().Head().Content())); + else if ("Double" == type) + writer.Write(FromString<double>(pair.Tail().Head().Content())); + else if ("Int8" == type) + writer.Write(FromString<i8>(pair.Tail().Head().Content())); + else if ("Uint8" == type) + writer.Write(FromString<ui8>(pair.Tail().Head().Content())); + else if ("Int16" == type) + writer.Write(FromString<i16>(pair.Tail().Head().Content())); + else if ("Uint16" == type) + writer.Write(FromString<ui16>(pair.Tail().Head().Content())); + else if ("Int32" == type) + writer.Write(FromString<i32>(pair.Tail().Head().Content())); + else if ("Uint32" == type) + writer.Write(FromString<ui32>(pair.Tail().Head().Content())); + else if ("Int64" == type) + writer.Write(FromString<i64>(pair.Tail().Head().Content())); + else if ("Uint64" == type) + writer.Write(FromString<ui64>(pair.Tail().Head().Content())); + else + writer.Write(pair.Tail().Head().Content()); + + } + }); + writer.CloseMap(); + writer.Flush(); + if (json == "{}") + json.clear(); + return {std::move(json), std::move(compression)}; + } + } + } + return {TString(), TString()}; +} +} + +TRuntimeNode BuildParseCall( + TRuntimeNode input, + const std::string_view& format, + const std::string_view& compression, + TType* inputItemType, + TType* outputItemType, + NCommon::TMkqlBuildContext& ctx) +{ + if (!compression.empty()) { + input = ctx.ProgramBuilder.Map(input, [&ctx, &compression](TRuntimeNode item) { + return ctx.ProgramBuilder.Apply(ctx.ProgramBuilder.Udf(std::string("Decompress.") += compression), {item}); + }); + } + + const auto structType = static_cast<const TStructType*>(outputItemType); + if (format == "raw") { + MKQL_ENSURE(1U == structType->GetMembersCount(), "Expected single column."); + bool isOptional; + const auto schemeType = UnpackOptionalData(structType->GetMemberType(0U), isOptional)->GetSchemeType(); return ctx.ProgramBuilder.ExpandMap(ctx.ProgramBuilder.ToFlow(input), [&](TRuntimeNode item)->TRuntimeNode::TList { return { NUdf::TDataType<const char*>::Id == schemeType ? @@ -120,43 +120,43 @@ TRuntimeNode BuildParseCall( } else { const auto userType = ctx.ProgramBuilder.NewTupleType({ctx.ProgramBuilder.NewTupleType({inputItemType}), ctx.ProgramBuilder.NewStructType({}), outputItemType}); input = ctx.ProgramBuilder.ToFlow(ctx.ProgramBuilder.Apply(ctx.ProgramBuilder.Udf("ClickHouseClient.ParseFormat", {}, userType, format), {input})); - } - + } + return ctx.ProgramBuilder.ExpandMap(input, - [&](TRuntimeNode item) { - TRuntimeNode::TList fields; - fields.reserve(structType->GetMembersCount()); - auto j = 0U; - std::generate_n(std::back_inserter(fields), structType->GetMembersCount(), [&](){ return ctx.ProgramBuilder.Member(item, structType->GetMemberName(j++)); }); - return fields; - }); -} - -NCommon::IMkqlCallableCompiler::TCompiler BuildCompiler(const std::string_view& providerName) { - return [providerName](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) { - if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == providerName) { - const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx); - const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *wrapper.Input().Ref().GetTypeAnn(), ctx.ProgramBuilder); - const auto outputItemType = NCommon::BuildType(wrapper.RowType().Ref(), *wrapper.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder); - const auto& settings = GetSettings(wrapper.Settings().Cast().Ref()); - return BuildParseCall(input, GetFormat(wrapper.Settings().Cast().Ref()).Content() + settings.front(), settings.back(), inputItemType, outputItemType, ctx); - } - - return TRuntimeNode(); - }; -} - -TMaybe<TRuntimeNode> TryWrapWithParser(const TDqSourceWideWrap& wrapper, NCommon::TMkqlBuildContext& ctx) { - const auto& format = GetFormat(wrapper.Settings().Cast().Ref()); - if (!format.Content()) { - return TMaybe<TRuntimeNode>(); - } - - const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx); - const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *wrapper.Input().Ref().GetTypeAnn(), ctx.ProgramBuilder); - const auto outputItemType = NCommon::BuildType(wrapper.RowType().Ref(), *wrapper.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder); - const auto& settings = GetSettings(wrapper.Settings().Cast().Ref()); - return BuildParseCall(input, format.Content() + settings.front(), settings.back(), inputItemType, outputItemType, ctx); -} - -} + [&](TRuntimeNode item) { + TRuntimeNode::TList fields; + fields.reserve(structType->GetMembersCount()); + auto j = 0U; + std::generate_n(std::back_inserter(fields), structType->GetMembersCount(), [&](){ return ctx.ProgramBuilder.Member(item, structType->GetMemberName(j++)); }); + return fields; + }); +} + +NCommon::IMkqlCallableCompiler::TCompiler BuildCompiler(const std::string_view& providerName) { + return [providerName](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) { + if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == providerName) { + const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx); + const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *wrapper.Input().Ref().GetTypeAnn(), ctx.ProgramBuilder); + const auto outputItemType = NCommon::BuildType(wrapper.RowType().Ref(), *wrapper.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder); + const auto& settings = GetSettings(wrapper.Settings().Cast().Ref()); + return BuildParseCall(input, GetFormat(wrapper.Settings().Cast().Ref()).Content() + settings.front(), settings.back(), inputItemType, outputItemType, ctx); + } + + return TRuntimeNode(); + }; +} + +TMaybe<TRuntimeNode> TryWrapWithParser(const TDqSourceWideWrap& wrapper, NCommon::TMkqlBuildContext& ctx) { + const auto& format = GetFormat(wrapper.Settings().Cast().Ref()); + if (!format.Content()) { + return TMaybe<TRuntimeNode>(); + } + + const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx); + const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *wrapper.Input().Ref().GetTypeAnn(), ctx.ProgramBuilder); + const auto outputItemType = NCommon::BuildType(wrapper.RowType().Ref(), *wrapper.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder); + const auto& settings = GetSettings(wrapper.Settings().Cast().Ref()); + return BuildParseCall(input, format.Content() + settings.front(), settings.back(), inputItemType, outputItemType, ctx); +} + +} diff --git a/ydb/library/yql/providers/common/mkql/parser.h b/ydb/library/yql/providers/common/mkql/parser.h index 1a6e821c179..285366a3867 100644 --- a/ydb/library/yql/providers/common/mkql/parser.h +++ b/ydb/library/yql/providers/common/mkql/parser.h @@ -1,26 +1,26 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/ast/yql_expr.h> - + #include <ydb/library/yql/core/yql_opt_utils.h> #include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h> #include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h> #include <ydb/library/yql/providers/common/mkql/yql_provider_mkql.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_node.h> - -#include <algorithm> - -namespace NYql { - -NKikimr::NMiniKQL::TRuntimeNode BuildParseCall( - NKikimr::NMiniKQL::TRuntimeNode input, - const std::string_view& format, - const std::string_view& compression, - NKikimr::NMiniKQL::TType* inputItemType, - NKikimr::NMiniKQL::TType* outputItemType, - NCommon::TMkqlBuildContext& ctx); - -TMaybe<NKikimr::NMiniKQL::TRuntimeNode> TryWrapWithParser(const NYql::NNodes::TDqSourceWideWrap& wrapper, NCommon::TMkqlBuildContext& ctx); - -} + +#include <algorithm> + +namespace NYql { + +NKikimr::NMiniKQL::TRuntimeNode BuildParseCall( + NKikimr::NMiniKQL::TRuntimeNode input, + const std::string_view& format, + const std::string_view& compression, + NKikimr::NMiniKQL::TType* inputItemType, + NKikimr::NMiniKQL::TType* outputItemType, + NCommon::TMkqlBuildContext& ctx); + +TMaybe<NKikimr::NMiniKQL::TRuntimeNode> TryWrapWithParser(const NYql::NNodes::TDqSourceWideWrap& wrapper, NCommon::TMkqlBuildContext& ctx); + +} diff --git a/ydb/library/yql/providers/common/mkql/ya.make b/ydb/library/yql/providers/common/mkql/ya.make index 1c53310c007..8de943fad2e 100644 --- a/ydb/library/yql/providers/common/mkql/ya.make +++ b/ydb/library/yql/providers/common/mkql/ya.make @@ -7,8 +7,8 @@ OWNER( ) SRCS( - parser.cpp - parser.h + parser.cpp + parser.h yql_provider_mkql.cpp yql_provider_mkql.h yql_type_mkql.cpp @@ -16,7 +16,7 @@ SRCS( ) PEERDIR( - library/cpp/json + library/cpp/json ydb/library/yql/ast ydb/library/yql/minikql ydb/library/yql/public/decimal @@ -30,5 +30,5 @@ PEERDIR( YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp index bde9e215b62..dd1f162d784 100644 --- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp +++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp @@ -1569,75 +1569,75 @@ TMkqlCommonCallableCompiler::TShared::TShared() { AddCallable("HoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto stream = MkqlBuildExpr(node.Head(), ctx); - const auto timeExtractor = [&](TRuntimeNode item) { + const auto timeExtractor = [&](TRuntimeNode item) { return MkqlBuildLambda(*node.Child(1), ctx, {item}); }; - const auto hop = MkqlBuildExpr(*node.Child(2), ctx); - const auto interval = MkqlBuildExpr(*node.Child(3), ctx); - const auto delay = MkqlBuildExpr(*node.Child(4), ctx); - - const auto init = [&](TRuntimeNode item) { - return MkqlBuildLambda(*node.Child(5), ctx, {item}); - }; + const auto hop = MkqlBuildExpr(*node.Child(2), ctx); + const auto interval = MkqlBuildExpr(*node.Child(3), ctx); + const auto delay = MkqlBuildExpr(*node.Child(4), ctx); + + const auto init = [&](TRuntimeNode item) { + return MkqlBuildLambda(*node.Child(5), ctx, {item}); + }; const auto update = [&](TRuntimeNode item, TRuntimeNode state) { - return MkqlBuildLambda(*node.Child(6), ctx, {item, state}); + return MkqlBuildLambda(*node.Child(6), ctx, {item, state}); }; const auto save = node.Child(3)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) { - return MkqlBuildLambda(*node.Child(7), ctx, {state}); + return MkqlBuildLambda(*node.Child(7), ctx, {state}); }; const auto load = node.Child(4)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) { - return MkqlBuildLambda(*node.Child(8), ctx, {state}); + return MkqlBuildLambda(*node.Child(8), ctx, {state}); }; const auto merge = [&](TRuntimeNode state1, TRuntimeNode state2) { - return MkqlBuildLambda(*node.Child(9), ctx, {state1, state2}); + return MkqlBuildLambda(*node.Child(9), ctx, {state1, state2}); }; const auto finish = [&](TRuntimeNode state, TRuntimeNode time) { - return MkqlBuildLambda(*node.Child(10), ctx, {state, time}); + return MkqlBuildLambda(*node.Child(10), ctx, {state, time}); }; return ctx.ProgramBuilder.HoppingCore( stream, timeExtractor, init, update, save, load, merge, finish, hop, interval, delay); }); - AddCallable("MultiHoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) { - const auto stream = MkqlBuildExpr(node.Head(), ctx); - - const auto keyExtractor = [&](TRuntimeNode item) { - return MkqlBuildLambda(*node.Child(1), ctx, {item}); - }; - - const auto timeExtractor = [&](TRuntimeNode item) { - return MkqlBuildLambda(*node.Child(2), ctx, {item}); - }; - const auto hop = MkqlBuildExpr(*node.Child(3), ctx); - const auto interval = MkqlBuildExpr(*node.Child(4), ctx); - const auto delay = MkqlBuildExpr(*node.Child(5), ctx); - const auto dataWatermarks = ctx.ProgramBuilder.NewDataLiteral(FromString<bool>(*node.Child(6), NUdf::EDataSlot::Bool)); - - const auto init = [&](TRuntimeNode item) { - return MkqlBuildLambda(*node.Child(7), ctx, {item}); - }; - const auto update = [&](TRuntimeNode item, TRuntimeNode state) { - return MkqlBuildLambda(*node.Child(8), ctx, {item, state}); - }; - const auto save = node.Child(3)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) { - return MkqlBuildLambda(*node.Child(9), ctx, {state}); - }; - const auto load = node.Child(4)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) { - return MkqlBuildLambda(*node.Child(10), ctx, {state}); - }; - const auto merge = [&](TRuntimeNode state1, TRuntimeNode state2) { - return MkqlBuildLambda(*node.Child(11), ctx, {state1, state2}); - }; - const auto finish = [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { - return MkqlBuildLambda(*node.Child(12), ctx, {key, state, time}); - }; - - return ctx.ProgramBuilder.MultiHoppingCore( - stream, keyExtractor, timeExtractor, init, update, save, load, merge, finish, - hop, interval, delay, dataWatermarks); - }); - + AddCallable("MultiHoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) { + const auto stream = MkqlBuildExpr(node.Head(), ctx); + + const auto keyExtractor = [&](TRuntimeNode item) { + return MkqlBuildLambda(*node.Child(1), ctx, {item}); + }; + + const auto timeExtractor = [&](TRuntimeNode item) { + return MkqlBuildLambda(*node.Child(2), ctx, {item}); + }; + const auto hop = MkqlBuildExpr(*node.Child(3), ctx); + const auto interval = MkqlBuildExpr(*node.Child(4), ctx); + const auto delay = MkqlBuildExpr(*node.Child(5), ctx); + const auto dataWatermarks = ctx.ProgramBuilder.NewDataLiteral(FromString<bool>(*node.Child(6), NUdf::EDataSlot::Bool)); + + const auto init = [&](TRuntimeNode item) { + return MkqlBuildLambda(*node.Child(7), ctx, {item}); + }; + const auto update = [&](TRuntimeNode item, TRuntimeNode state) { + return MkqlBuildLambda(*node.Child(8), ctx, {item, state}); + }; + const auto save = node.Child(3)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) { + return MkqlBuildLambda(*node.Child(9), ctx, {state}); + }; + const auto load = node.Child(4)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) { + return MkqlBuildLambda(*node.Child(10), ctx, {state}); + }; + const auto merge = [&](TRuntimeNode state1, TRuntimeNode state2) { + return MkqlBuildLambda(*node.Child(11), ctx, {state1, state2}); + }; + const auto finish = [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { + return MkqlBuildLambda(*node.Child(12), ctx, {key, state, time}); + }; + + return ctx.ProgramBuilder.MultiHoppingCore( + stream, keyExtractor, timeExtractor, init, update, save, load, merge, finish, + hop, interval, delay, dataWatermarks); + }); + AddCallable("ToDict", [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto list = MkqlBuildExpr(node.Head(), ctx); TMaybe<bool> isMany; diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 4f36ed8a710..8561fe25368 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -334,25 +334,25 @@ message TS3GatewayConfig { ///////////////////////////// Solomon ///////////////////////////// message TSolomonClusterConfig { - enum ESolomonClusterType { - SCT_UNSPECIFIED = 0; - SCT_SOLOMON = 1; - SCT_MONITORING = 2; - } - - message TShardPath { - required string Project = 1; // CloudId for YC - required string Cluster = 2; // FolderId for YC - } - + enum ESolomonClusterType { + SCT_UNSPECIFIED = 0; + SCT_SOLOMON = 1; + SCT_MONITORING = 2; + } + + message TShardPath { + required string Project = 1; // CloudId for YC + required string Cluster = 2; // FolderId for YC + } + optional string Name = 1; optional string Cluster = 2; optional bool UseSsl = 3; - optional ESolomonClusterType ClusterType = 4 [default = SCT_SOLOMON]; + optional ESolomonClusterType ClusterType = 4 [default = SCT_SOLOMON]; optional string Token = 5; optional string ServiceAccountId = 6; optional string ServiceAccountIdSignature = 7; - optional TShardPath Path = 8; + optional TShardPath Path = 8; repeated TAttr Settings = 100; } diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index 96c5076fd0f..aa782ee93ff 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -574,16 +574,16 @@ static void GetToken(const TString& string, TString& out, const TTypeAnnotationC } } } - for (auto& x : type.DataSinks) { - auto tokens = x->GetClusterTokens(); - if (tokens) { - auto token = tokens->FindPtr(clusterName); - if (token) { - out = *token; - return; - } - } - } + for (auto& x : type.DataSinks) { + auto tokens = x->GetClusterTokens(); + if (tokens) { + auto token = tokens->FindPtr(clusterName); + if (token) { + out = *token; + return; + } + } + } } } diff --git a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp index 97592bac3e0..39cae924364 100644 --- a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp +++ b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp @@ -1,143 +1,143 @@ -#include "dq_fake_ca.h" - +#include "dq_fake_ca.h" + #include <ydb/library/yql/minikql/mkql_string_util.h> - + #include <ydb/core/testlib/basics/appdata.h> - -#include <util/system/env.h> - -#include <condition_variable> -#include <thread> - -namespace NYql::NDq { - -using namespace NActors; - -NYql::NDqProto::TCheckpoint CreateCheckpoint(ui64 id) { - auto checkpoint = NYql::NDqProto::TCheckpoint(); - checkpoint.SetGeneration(0); - checkpoint.SetId(id); - return checkpoint; -} - -TFakeActor::TFakeActor(TSourcePromises& sourcePromises, TSinkPromises& sinkPromises) - : TActor<TFakeActor>(&TFakeActor::StateFunc) - , MemoryInfo("test") - , HolderFactory(Alloc.Ref(), MemoryInfo) - , SourceCallbacks(*this) - , SinkCallbacks(*this) - , SourcePromises(sourcePromises) - , SinkPromises(sinkPromises) -{ - Alloc.Release(); -} - -TFakeActor::~TFakeActor() { - Alloc.Acquire(); -} - -void TFakeActor::InitSink(IDqSinkActor* dqSink, IActor* dqSinkAsActor) { - DqSinkActorId = RegisterWithSameMailbox(dqSinkAsActor), - DqSinkActor = dqSink; - DqSinkActorAsActor = dqSinkAsActor; -} - -void TFakeActor::InitSource(IDqSourceActor* dqSource, IActor* dqSourceAsActor) { - DqSourceActorId = RegisterWithSameMailbox(dqSourceAsActor), - DqSourceActor = dqSource; - DqSourceActorAsActor = dqSourceAsActor; -} - -void TFakeActor::Terminate() { - if (DqSourceActorId) { + +#include <util/system/env.h> + +#include <condition_variable> +#include <thread> + +namespace NYql::NDq { + +using namespace NActors; + +NYql::NDqProto::TCheckpoint CreateCheckpoint(ui64 id) { + auto checkpoint = NYql::NDqProto::TCheckpoint(); + checkpoint.SetGeneration(0); + checkpoint.SetId(id); + return checkpoint; +} + +TFakeActor::TFakeActor(TSourcePromises& sourcePromises, TSinkPromises& sinkPromises) + : TActor<TFakeActor>(&TFakeActor::StateFunc) + , MemoryInfo("test") + , HolderFactory(Alloc.Ref(), MemoryInfo) + , SourceCallbacks(*this) + , SinkCallbacks(*this) + , SourcePromises(sourcePromises) + , SinkPromises(sinkPromises) +{ + Alloc.Release(); +} + +TFakeActor::~TFakeActor() { + Alloc.Acquire(); +} + +void TFakeActor::InitSink(IDqSinkActor* dqSink, IActor* dqSinkAsActor) { + DqSinkActorId = RegisterWithSameMailbox(dqSinkAsActor), + DqSinkActor = dqSink; + DqSinkActorAsActor = dqSinkAsActor; +} + +void TFakeActor::InitSource(IDqSourceActor* dqSource, IActor* dqSourceAsActor) { + DqSourceActorId = RegisterWithSameMailbox(dqSourceAsActor), + DqSourceActor = dqSource; + DqSourceActorAsActor = dqSourceAsActor; +} + +void TFakeActor::Terminate() { + if (DqSourceActorId) { DqSourceActor->PassAway(); - - DqSourceActorId = std::nullopt; - DqSourceActor = nullptr; - DqSourceActorAsActor = nullptr; - } - - if (DqSinkActorId) { + + DqSourceActorId = std::nullopt; + DqSourceActor = nullptr; + DqSourceActorAsActor = nullptr; + } + + if (DqSinkActorId) { DqSinkActor->PassAway(); - - DqSinkActorId = std::nullopt; - DqSinkActor = nullptr; - DqSinkActorAsActor = nullptr; - } -} - -TFakeActor::TSourceCallbacks& TFakeActor::GetSourceCallbacks() { - return SourceCallbacks; -} - -TFakeActor::TSinkCallbacks& TFakeActor::GetSinkCallbacks() { - return SinkCallbacks; -} - -NKikimr::NMiniKQL::THolderFactory& TFakeActor::GetHolderFactory() { - return HolderFactory; -} - -TFakeCASetup::TFakeCASetup() - : Runtime(new NActors::TTestBasicRuntime(1, true)) - , FakeActorId(0, "FakeActor") -{ - Runtime->AddLocalService( - FakeActorId, - NActors::TActorSetupCmd( - new TFakeActor(SourcePromises, SinkPromises), - NActors::TMailboxType::Simple, - 0)); - - TAutoPtr<NKikimr::TAppPrepare> app = new NKikimr::TAppPrepare(); - Runtime->Initialize(app->Unwrap()); -} - -TFakeCASetup::~TFakeCASetup() { - Execute([](TFakeActor& actor) { - actor.Terminate(); - }); -} - -void TFakeCASetup::SinkWrite(const TWriteValueProducer valueProducer, TMaybe<NDqProto::TCheckpoint> checkpoint) { - Execute([&valueProducer, checkpoint](TFakeActor& actor) { - auto batch = valueProducer(actor.GetHolderFactory()); - Y_ASSERT(actor.DqSinkActor); - actor.DqSinkActor->SendData(std::move(batch), 0, checkpoint, false); - }); -} - + + DqSinkActorId = std::nullopt; + DqSinkActor = nullptr; + DqSinkActorAsActor = nullptr; + } +} + +TFakeActor::TSourceCallbacks& TFakeActor::GetSourceCallbacks() { + return SourceCallbacks; +} + +TFakeActor::TSinkCallbacks& TFakeActor::GetSinkCallbacks() { + return SinkCallbacks; +} + +NKikimr::NMiniKQL::THolderFactory& TFakeActor::GetHolderFactory() { + return HolderFactory; +} + +TFakeCASetup::TFakeCASetup() + : Runtime(new NActors::TTestBasicRuntime(1, true)) + , FakeActorId(0, "FakeActor") +{ + Runtime->AddLocalService( + FakeActorId, + NActors::TActorSetupCmd( + new TFakeActor(SourcePromises, SinkPromises), + NActors::TMailboxType::Simple, + 0)); + + TAutoPtr<NKikimr::TAppPrepare> app = new NKikimr::TAppPrepare(); + Runtime->Initialize(app->Unwrap()); +} + +TFakeCASetup::~TFakeCASetup() { + Execute([](TFakeActor& actor) { + actor.Terminate(); + }); +} + +void TFakeCASetup::SinkWrite(const TWriteValueProducer valueProducer, TMaybe<NDqProto::TCheckpoint> checkpoint) { + Execute([&valueProducer, checkpoint](TFakeActor& actor) { + auto batch = valueProducer(actor.GetHolderFactory()); + Y_ASSERT(actor.DqSinkActor); + actor.DqSinkActor->SendData(std::move(batch), 0, checkpoint, false); + }); +} + void TFakeCASetup::SaveSourceState(NDqProto::TCheckpoint checkpoint, NDqProto::TSourceState& state) { - Execute([&state, &checkpoint](TFakeActor& actor) { - Y_ASSERT(actor.DqSourceActor); + Execute([&state, &checkpoint](TFakeActor& actor) { + Y_ASSERT(actor.DqSourceActor); actor.DqSourceActor->SaveState(checkpoint, state); - }); -} - + }); +} + void TFakeCASetup::LoadSource(const NDqProto::TSourceState& state) { - Execute([&state](TFakeActor& actor) { - Y_ASSERT(actor.DqSourceActor); - actor.DqSourceActor->LoadState(state); - }); -} - + Execute([&state](TFakeActor& actor) { + Y_ASSERT(actor.DqSourceActor); + actor.DqSourceActor->LoadState(state); + }); +} + void TFakeCASetup::LoadSink(const NDqProto::TSinkState& state) { - Execute([&state](TFakeActor& actor) { - Y_ASSERT(actor.DqSinkActor); - actor.DqSinkActor->LoadState(state); - }); -} - -void TFakeCASetup::Execute(TCallback callback) { - std::exception_ptr exception_ptr = nullptr; - const TActorId& edgeId = Runtime->AllocateEdgeActor(); - auto promise = NThreading::NewPromise(); - Runtime->Send(new IEventHandle(FakeActorId, edgeId, new TEvPrivate::TEvExecute(promise, callback, exception_ptr))); - auto future = promise.GetFuture(); - future.Wait(); - if (exception_ptr) { - std::rethrow_exception(exception_ptr); - } -} - -} // namespace NKikimr::NMiniKQL + Execute([&state](TFakeActor& actor) { + Y_ASSERT(actor.DqSinkActor); + actor.DqSinkActor->LoadState(state); + }); +} + +void TFakeCASetup::Execute(TCallback callback) { + std::exception_ptr exception_ptr = nullptr; + const TActorId& edgeId = Runtime->AllocateEdgeActor(); + auto promise = NThreading::NewPromise(); + Runtime->Send(new IEventHandle(FakeActorId, edgeId, new TEvPrivate::TEvExecute(promise, callback, exception_ptr))); + auto future = promise.GetFuture(); + future.Wait(); + if (exception_ptr) { + std::rethrow_exception(exception_ptr); + } +} + +} // namespace NKikimr::NMiniKQL diff --git a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h index 96fca4f4650..94d815a1eb8 100644 --- a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h +++ b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h @@ -1,223 +1,223 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> #include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> #include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h> #include <ydb/library/yql/minikql/mkql_alloc.h> - + #include <ydb/core/testlib/basics/runtime.h> - -#include <library/cpp/retry/retry.h> -#include <library/cpp/testing/unittest/registar.h> - -#include <chrono> -#include <queue> - -namespace NYql::NDq { - -class TFakeActor; - -using TRuntimePtr = std::unique_ptr<NActors::TTestActorRuntime>; -using TCallback = std::function<void(TFakeActor&)>; -template<typename T> -using TReadValueParser = std::function<std::vector<T>(const NUdf::TUnboxedValue&)>; -using TWriteValueProducer = std::function<NKikimr::NMiniKQL::TUnboxedValueVector(NKikimr::NMiniKQL::THolderFactory&)>; - -namespace { - struct TEvPrivate { - // Event ids - enum EEv : ui32 { - EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), - - EvExecute = EvBegin, - - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)"); - - // Events - - struct TEvExecute : public NActors::TEventLocal<TEvExecute, EvExecute> { - TEvExecute(NThreading::TPromise<void>& promise, TCallback callback, std::exception_ptr& resultException) - : Promise(promise) - , Callback(callback) - , ResultException(resultException) - {} - - NThreading::TPromise<void> Promise; - TCallback Callback; - std::exception_ptr& ResultException; - }; - }; -} - -struct TSourcePromises { - NThreading::TPromise<void> NewSourceDataArrived = NThreading::NewPromise(); - NThreading::TPromise<TIssues> FatalError = NThreading::NewPromise<TIssues>(); -}; - -struct TSinkPromises { - NThreading::TPromise<void> ResumeExecution = NThreading::NewPromise(); - NThreading::TPromise<TIssues> Issue = NThreading::NewPromise<TIssues>(); + +#include <library/cpp/retry/retry.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <chrono> +#include <queue> + +namespace NYql::NDq { + +class TFakeActor; + +using TRuntimePtr = std::unique_ptr<NActors::TTestActorRuntime>; +using TCallback = std::function<void(TFakeActor&)>; +template<typename T> +using TReadValueParser = std::function<std::vector<T>(const NUdf::TUnboxedValue&)>; +using TWriteValueProducer = std::function<NKikimr::NMiniKQL::TUnboxedValueVector(NKikimr::NMiniKQL::THolderFactory&)>; + +namespace { + struct TEvPrivate { + // Event ids + enum EEv : ui32 { + EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), + + EvExecute = EvBegin, + + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)"); + + // Events + + struct TEvExecute : public NActors::TEventLocal<TEvExecute, EvExecute> { + TEvExecute(NThreading::TPromise<void>& promise, TCallback callback, std::exception_ptr& resultException) + : Promise(promise) + , Callback(callback) + , ResultException(resultException) + {} + + NThreading::TPromise<void> Promise; + TCallback Callback; + std::exception_ptr& ResultException; + }; + }; +} + +struct TSourcePromises { + NThreading::TPromise<void> NewSourceDataArrived = NThreading::NewPromise(); + NThreading::TPromise<TIssues> FatalError = NThreading::NewPromise<TIssues>(); +}; + +struct TSinkPromises { + NThreading::TPromise<void> ResumeExecution = NThreading::NewPromise(); + NThreading::TPromise<TIssues> Issue = NThreading::NewPromise<TIssues>(); NThreading::TPromise<NDqProto::TSinkState> StateSaved = NThreading::NewPromise<NDqProto::TSinkState>(); -}; - -NYql::NDqProto::TCheckpoint CreateCheckpoint(ui64 id = 0); - -class TFakeActor : public NActors::TActor<TFakeActor> { - struct TSourceCallbacks : public IDqSourceActor::ICallbacks { - explicit TSourceCallbacks(TFakeActor& parent) : Parent(parent) {} - - void OnNewSourceDataArrived(ui64) override { - Parent.SourcePromises.NewSourceDataArrived.SetValue(); - Parent.SourcePromises.NewSourceDataArrived = NThreading::NewPromise(); - } - - void OnSourceError(ui64, const TIssues& issues, bool isFatal) override { - Y_UNUSED(isFatal); - Parent.SourcePromises.FatalError.SetValue(issues); - Parent.SourcePromises.FatalError = NThreading::NewPromise<TIssues>(); - } - - TFakeActor& Parent; - }; - - struct TSinkCallbacks : public IDqSinkActor::ICallbacks { - explicit TSinkCallbacks(TFakeActor& parent) : Parent(parent) {} - - void ResumeExecution() override { - Parent.SinkPromises.ResumeExecution.SetValue(); - Parent.SinkPromises.ResumeExecution = NThreading::NewPromise(); - }; - - void OnSinkError(ui64, const TIssues& issues, bool isFatal) override { - Y_UNUSED(isFatal); - Parent.SinkPromises.Issue.SetValue(issues); - Parent.SinkPromises.Issue = NThreading::NewPromise<TIssues>(); - }; - +}; + +NYql::NDqProto::TCheckpoint CreateCheckpoint(ui64 id = 0); + +class TFakeActor : public NActors::TActor<TFakeActor> { + struct TSourceCallbacks : public IDqSourceActor::ICallbacks { + explicit TSourceCallbacks(TFakeActor& parent) : Parent(parent) {} + + void OnNewSourceDataArrived(ui64) override { + Parent.SourcePromises.NewSourceDataArrived.SetValue(); + Parent.SourcePromises.NewSourceDataArrived = NThreading::NewPromise(); + } + + void OnSourceError(ui64, const TIssues& issues, bool isFatal) override { + Y_UNUSED(isFatal); + Parent.SourcePromises.FatalError.SetValue(issues); + Parent.SourcePromises.FatalError = NThreading::NewPromise<TIssues>(); + } + + TFakeActor& Parent; + }; + + struct TSinkCallbacks : public IDqSinkActor::ICallbacks { + explicit TSinkCallbacks(TFakeActor& parent) : Parent(parent) {} + + void ResumeExecution() override { + Parent.SinkPromises.ResumeExecution.SetValue(); + Parent.SinkPromises.ResumeExecution = NThreading::NewPromise(); + }; + + void OnSinkError(ui64, const TIssues& issues, bool isFatal) override { + Y_UNUSED(isFatal); + Parent.SinkPromises.Issue.SetValue(issues); + Parent.SinkPromises.Issue = NThreading::NewPromise<TIssues>(); + }; + void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint&) override { - Y_UNUSED(outputIndex); - Parent.SinkPromises.StateSaved.SetValue(state); + Y_UNUSED(outputIndex); + Parent.SinkPromises.StateSaved.SetValue(state); Parent.SinkPromises.StateSaved = NThreading::NewPromise<NDqProto::TSinkState>(); - }; - - TFakeActor& Parent; - }; - -public: - TFakeActor(TSourcePromises& sourcePromises, TSinkPromises& sinkPromises); - ~TFakeActor(); - - void InitSink(IDqSinkActor* dqSink, IActor* dqSinkAsActor); - void InitSource(IDqSourceActor* dqSource, IActor* dqSourceAsActor); - void Terminate(); - - TSourceCallbacks& GetSourceCallbacks(); - TSinkCallbacks& GetSinkCallbacks(); - NKikimr::NMiniKQL::THolderFactory& GetHolderFactory(); - -public: - IDqSourceActor* DqSourceActor = nullptr; - IDqSinkActor* DqSinkActor = nullptr; - -private: - STRICT_STFUNC(StateFunc, - hFunc(TEvPrivate::TEvExecute, Handle); - ) - - void Handle(TEvPrivate::TEvExecute::TPtr& ev) { - TGuard<NKikimr::NMiniKQL::TScopedAlloc> guard(Alloc); - try { - ev->Get()->Callback(*this); - } catch (...) { - ev->Get()->ResultException = std::current_exception(); - } - ev->Get()->Promise.SetValue(); - } - -private: - NKikimr::NMiniKQL::TScopedAlloc Alloc; - NKikimr::NMiniKQL::TMemoryUsageInfo MemoryInfo; - NKikimr::NMiniKQL::THolderFactory HolderFactory; - - std::optional<NActors::TActorId> DqSourceActorId; - IActor* DqSourceActorAsActor = nullptr; - - std::optional<NActors::TActorId> DqSinkActorId; - IActor* DqSinkActorAsActor = nullptr; - - TSourceCallbacks SourceCallbacks; - TSinkCallbacks SinkCallbacks; - - TSourcePromises& SourcePromises; - TSinkPromises& SinkPromises; -}; - -struct TFakeCASetup { - TFakeCASetup(); - ~TFakeCASetup(); - - template<typename T> - std::vector<T> SourceRead(const TReadValueParser<T> parser, i64 freeSpace = 12345) { - std::vector<T> result; - Execute([&result, &parser, freeSpace](TFakeActor& actor) { - NKikimr::NMiniKQL::TUnboxedValueVector buffer; - bool finished = false; - actor.DqSourceActor->GetSourceData(buffer, finished, freeSpace); - - for (const auto& uv : buffer) { - for (const auto item : parser(uv)) { - result.emplace_back(item); - } - } - }); - - return result; - } - - template<typename T> - std::vector<T> SourceReadUntil( - const TReadValueParser<T> parser, - ui64 size, - i64 eachReadFreeSpace = 1000, - TDuration timeout = TDuration::Seconds(10)) - { - std::vector<T> result; - DoWithRetry([&](){ - auto batch = SourceRead<T>(parser, eachReadFreeSpace); - for (const auto& item : batch) { - result.emplace_back(item); - } - - if (result.size() < size) { - SourcePromises.NewSourceDataArrived.GetFuture().Wait(timeout); - ythrow yexception() << "Not enough data"; - } - }, - TRetryOptions(3), - false); - - return result; - } - - void SinkWrite(const TWriteValueProducer valueProducer, TMaybe<NDqProto::TCheckpoint> checkpoint = Nothing()); - + }; + + TFakeActor& Parent; + }; + +public: + TFakeActor(TSourcePromises& sourcePromises, TSinkPromises& sinkPromises); + ~TFakeActor(); + + void InitSink(IDqSinkActor* dqSink, IActor* dqSinkAsActor); + void InitSource(IDqSourceActor* dqSource, IActor* dqSourceAsActor); + void Terminate(); + + TSourceCallbacks& GetSourceCallbacks(); + TSinkCallbacks& GetSinkCallbacks(); + NKikimr::NMiniKQL::THolderFactory& GetHolderFactory(); + +public: + IDqSourceActor* DqSourceActor = nullptr; + IDqSinkActor* DqSinkActor = nullptr; + +private: + STRICT_STFUNC(StateFunc, + hFunc(TEvPrivate::TEvExecute, Handle); + ) + + void Handle(TEvPrivate::TEvExecute::TPtr& ev) { + TGuard<NKikimr::NMiniKQL::TScopedAlloc> guard(Alloc); + try { + ev->Get()->Callback(*this); + } catch (...) { + ev->Get()->ResultException = std::current_exception(); + } + ev->Get()->Promise.SetValue(); + } + +private: + NKikimr::NMiniKQL::TScopedAlloc Alloc; + NKikimr::NMiniKQL::TMemoryUsageInfo MemoryInfo; + NKikimr::NMiniKQL::THolderFactory HolderFactory; + + std::optional<NActors::TActorId> DqSourceActorId; + IActor* DqSourceActorAsActor = nullptr; + + std::optional<NActors::TActorId> DqSinkActorId; + IActor* DqSinkActorAsActor = nullptr; + + TSourceCallbacks SourceCallbacks; + TSinkCallbacks SinkCallbacks; + + TSourcePromises& SourcePromises; + TSinkPromises& SinkPromises; +}; + +struct TFakeCASetup { + TFakeCASetup(); + ~TFakeCASetup(); + + template<typename T> + std::vector<T> SourceRead(const TReadValueParser<T> parser, i64 freeSpace = 12345) { + std::vector<T> result; + Execute([&result, &parser, freeSpace](TFakeActor& actor) { + NKikimr::NMiniKQL::TUnboxedValueVector buffer; + bool finished = false; + actor.DqSourceActor->GetSourceData(buffer, finished, freeSpace); + + for (const auto& uv : buffer) { + for (const auto item : parser(uv)) { + result.emplace_back(item); + } + } + }); + + return result; + } + + template<typename T> + std::vector<T> SourceReadUntil( + const TReadValueParser<T> parser, + ui64 size, + i64 eachReadFreeSpace = 1000, + TDuration timeout = TDuration::Seconds(10)) + { + std::vector<T> result; + DoWithRetry([&](){ + auto batch = SourceRead<T>(parser, eachReadFreeSpace); + for (const auto& item : batch) { + result.emplace_back(item); + } + + if (result.size() < size) { + SourcePromises.NewSourceDataArrived.GetFuture().Wait(timeout); + ythrow yexception() << "Not enough data"; + } + }, + TRetryOptions(3), + false); + + return result; + } + + void SinkWrite(const TWriteValueProducer valueProducer, TMaybe<NDqProto::TCheckpoint> checkpoint = Nothing()); + void SaveSourceState(NDqProto::TCheckpoint checkpoint, NDqProto::TSourceState& state); - + void LoadSource(const NDqProto::TSourceState& state); void LoadSink(const NDqProto::TSinkState& state); - - void Execute(TCallback callback); - -public: - TRuntimePtr Runtime; - NActors::TActorId FakeActorId; - TSourcePromises SourcePromises; - TSinkPromises SinkPromises; -}; - -} // namespace NKikimr::NMiniKQL + + void Execute(TCallback callback); + +public: + TRuntimePtr Runtime; + NActors::TActorId FakeActorId; + TSourcePromises SourcePromises; + TSinkPromises SinkPromises; +}; + +} // namespace NKikimr::NMiniKQL diff --git a/ydb/library/yql/providers/common/ut_helpers/ya.make b/ydb/library/yql/providers/common/ut_helpers/ya.make index cf95837bb36..14cb9b41a6e 100644 --- a/ydb/library/yql/providers/common/ut_helpers/ya.make +++ b/ydb/library/yql/providers/common/ut_helpers/ya.make @@ -1,21 +1,21 @@ -LIBRARY() - -OWNER(g:yql) - -SRCS( - dq_fake_ca.cpp -) - -PEERDIR( - library/cpp/retry +LIBRARY() + +OWNER(g:yql) + +SRCS( + dq_fake_ca.cpp +) + +PEERDIR( + library/cpp/retry ydb/core/testlib/basics ydb/library/yql/minikql ydb/library/yql/minikql/computation ydb/library/yql/public/udf/service/exception_policy ydb/library/yql/sql ydb/library/yql/providers/common/comp_nodes -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/dq/actors/task_controller.cpp b/ydb/library/yql/providers/dq/actors/task_controller.cpp index 17cf0a4c550..238828e453b 100644 --- a/ydb/library/yql/providers/dq/actors/task_controller.cpp +++ b/ydb/library/yql/providers/dq/actors/task_controller.cpp @@ -221,8 +221,8 @@ private: publicCounterName = "query.cpu_usage_us"; isDeriv = true; } else if (name == "Bytes") { - if (labels.find("Source") != labels.end()) publicCounterName = "query.input_bytes"; - else if (labels.find("Sink") != labels.end()) publicCounterName = "query.output_bytes"; + if (labels.find("Source") != labels.end()) publicCounterName = "query.input_bytes"; + else if (labels.find("Sink") != labels.end()) publicCounterName = "query.output_bytes"; isDeriv = true; } if (publicCounterName) { @@ -273,10 +273,10 @@ private: TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "MkqlMaxMemoryUsage"), v); } - for (const auto& stat : s.GetMkqlStats()) { - TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, stat.GetName()), stat.GetValue()); - } - + for (const auto& stat : s.GetMkqlStats()) { + TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, stat.GetName()), stat.GetValue()); + } + if (stats.ComputeCpuTimeByRunSize()) { auto& hist = TaskStat.GetHistogram(TaskStat.GetCounterName("TaskRunner", labels, "ComputeTimeByRunMs")); for (const auto& bucket : s.GetComputeCpuTimeByRun()) { @@ -340,44 +340,44 @@ private: // } } - for (const auto& stats : s.GetSources()) { - std::map<TString, TString> labels = { - {"Task", ToString(taskId)}, + for (const auto& stats : s.GetSources()) { + std::map<TString, TString> labels = { + {"Task", ToString(taskId)}, {"Source", ToString(stats.GetInputIndex())} - }; - - ADD_COUNTER(Chunks); - ADD_COUNTER(Bytes); - ADD_COUNTER(RowsIn); - ADD_COUNTER(RowsOut); - ADD_COUNTER(MaxMemoryUsage); - - ADD_COUNTER(ErrorsCount); - + }; + + ADD_COUNTER(Chunks); + ADD_COUNTER(Bytes); + ADD_COUNTER(RowsIn); + ADD_COUNTER(RowsOut); + ADD_COUNTER(MaxMemoryUsage); + + ADD_COUNTER(ErrorsCount); + // if (stats.GetFinishTs() >= stats.GetStartTs()) { // TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs()); // } - } - - for (const auto& stats : s.GetSinks()) { - std::map<TString, TString> labels = { - {"Task", ToString(taskId)}, + } + + for (const auto& stats : s.GetSinks()) { + std::map<TString, TString> labels = { + {"Task", ToString(taskId)}, {"Sink", ToString(stats.GetOutputIndex())} - }; - - ADD_COUNTER(Chunks) - ADD_COUNTER(Bytes); - ADD_COUNTER(RowsIn); - ADD_COUNTER(RowsOut); - ADD_COUNTER(MaxMemoryUsage); - - ADD_COUNTER(ErrorsCount); - + }; + + ADD_COUNTER(Chunks) + ADD_COUNTER(Bytes); + ADD_COUNTER(RowsIn); + ADD_COUNTER(RowsOut); + ADD_COUNTER(MaxMemoryUsage); + + ADD_COUNTER(ErrorsCount); + // if (stats.GetFinishTs() >= stats.GetStartTs()) { // TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs()); // } - } - + } + #undef ADD_COUNTER } @@ -441,13 +441,13 @@ private: YQL_LOG(DEBUG) << "Update channels"; for (const auto& [task, actorId] : Tasks) { auto ev = MakeHolder<NDq::TEvDqCompute::TEvChannelsInfo>(); - - for (const auto& input : task.GetInputs()) { - for (const auto& channel : input.GetChannels()) { - *ev->Record.AddUpdate() = channel; - } - } - + + for (const auto& input : task.GetInputs()) { + for (const auto& channel : input.GetChannels()) { + *ev->Record.AddUpdate() = channel; + } + } + for (const auto& output : task.GetOutputs()) { for (const auto& channel : output.GetChannels()) { *ev->Record.AddUpdate() = channel; diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp index c4933a31fbb..f9ba89a33d9 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp +++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp @@ -9,7 +9,7 @@ TDqConfiguration::TDqConfiguration() { REGISTER_SETTING(*this, MaxTasksPerOperation); REGISTER_SETTING(*this, WorkersPerOperation); REGISTER_SETTING(*this, MaxDataSizePerQuery); - REGISTER_SETTING(*this, AnalyticsHopping); + REGISTER_SETTING(*this, AnalyticsHopping); REGISTER_SETTING(*this, AnalyzeQuery); REGISTER_SETTING(*this, _AnalyzeQueryPercentage); REGISTER_SETTING(*this, MaxRetries); diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.h b/ydb/library/yql/providers/dq/common/yql_dq_settings.h index 7cb3642946a..3c37e4d48fd 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_settings.h +++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.h @@ -35,7 +35,7 @@ struct TDqSettings { NCommon::TConfSetting<ui32, false> MaxTasksPerOperation; NCommon::TConfSetting<ui32, false> WorkersPerOperation; NCommon::TConfSetting<ui64, false> MaxDataSizePerQuery; - NCommon::TConfSetting<bool, false> AnalyticsHopping; + NCommon::TConfSetting<bool, false> AnalyticsHopping; NCommon::TConfSetting<bool, false> AnalyzeQuery; NCommon::TConfSetting<int, false> _AnalyzeQueryPercentage; NCommon::TConfSetting<int, false> MaxRetries; @@ -78,7 +78,7 @@ struct TDqSettings { s->SetValue(ToString(*this->name.Get())); \ } - SAVE_SETTING(AnalyticsHopping); + SAVE_SETTING(AnalyticsHopping); SAVE_SETTING(MaxRetries); SAVE_SETTING(MaxNetworkRetries); SAVE_SETTING(WorkersPerOperation); diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp index 33e8883c7c4..9e87f81e7cf 100644 --- a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp +++ b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp @@ -15,35 +15,35 @@ using namespace NYql; using namespace NYql::NDq; using namespace NYql::NNodes; -namespace { - -constexpr TStringBuf YQL_TIME = "_yql_time"; - -TString BuildColumnName(const TExprBase column) { - if (const auto columnName = column.Maybe<TCoAtom>()) { - return columnName.Cast().StringValue(); - } - - if (const auto columnNames = column.Maybe<TCoAtomList>()) { - TStringBuilder columnNameBuilder; - for (const auto columnName : columnNames.Cast()) { - columnNameBuilder.append(columnName.StringValue()); - columnNameBuilder.append("_"); - } - return columnNameBuilder; - } - - YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: " - << column.Ptr()->Dump()); -} - -} - +namespace { + +constexpr TStringBuf YQL_TIME = "_yql_time"; + +TString BuildColumnName(const TExprBase column) { + if (const auto columnName = column.Maybe<TCoAtom>()) { + return columnName.Cast().StringValue(); + } + + if (const auto columnNames = column.Maybe<TCoAtomList>()) { + TStringBuilder columnNameBuilder; + for (const auto columnName : columnNames.Cast()) { + columnNameBuilder.append(columnName.StringValue()); + columnNameBuilder.append("_"); + } + return columnNameBuilder; + } + + YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: " + << column.Ptr()->Dump()); +} + +} + class TDqsLogicalOptProposalTransformer : public TOptimizeTransformerBase { public: - TDqsLogicalOptProposalTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config) + TDqsLogicalOptProposalTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config) : TOptimizeTransformerBase(typeCtx, NLog::EComponent::ProviderDq, {}) - , Config(config) + , Config(config) { #define HNDL(name) "DqsLogical-"#name, Hndl(&TDqsLogicalOptProposalTransformer::name) AddHandler(0, &TCoUnorderedBase::Match, HNDL(SkipUnordered)); @@ -74,15 +74,15 @@ protected: TMaybeNode<TExprBase> RewriteAggregate(TExprBase node, TExprContext& ctx) { auto aggregate = node.Cast<TCoAggregate>(); - auto input = aggregate.Input().Maybe<TDqConnection>(); - - auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping"); - if (input) { - if (hopSetting) { - return RewriteAsHoppingWindow(node, ctx, input.Cast()).Cast(); - } else { - return DqRewriteAggregate(node, ctx); - } + auto input = aggregate.Input().Maybe<TDqConnection>(); + + auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping"); + if (input) { + if (hopSetting) { + return RewriteAsHoppingWindow(node, ctx, input.Cast()).Cast(); + } else { + return DqRewriteAggregate(node, ctx); + } } return node; } @@ -117,7 +117,7 @@ protected: } return node; } - + TMaybeNode<TExprBase> MergeQueriesWithSinks(TExprBase node, TExprContext& ctx) { return DqMergeQueriesWithSinks(node, ctx); } @@ -126,661 +126,661 @@ protected: return DqUnorderedInStage(node, TDqReadWrapBase::Match, ctx, Types); } -private: - TMaybeNode<TExprBase> RewriteAsHoppingWindow(const TExprBase node, TExprContext& ctx, const TDqConnection& input) { - const auto aggregate = node.Cast<TCoAggregate>(); - const auto pos = aggregate.Pos(); - - YQL_CLOG(DEBUG, ProviderDq) << "OptimizeStreamingAggregate"; - - EnsureNotDistinct(aggregate); - - const auto aggregateInputType = GetSeqItemType(node.Ptr()->Head().GetTypeAnn())->Cast<TStructExprType>(); - TKeysDescription keysDescription(*aggregateInputType, aggregate.Keys()); - - if (keysDescription.NeedPickle()) { - return Build<TCoMap>(ctx, pos) - .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType)) - .Input<TCoAggregate>() - .InitFrom(aggregate) - .Input<TCoMap>() - .Lambda(keysDescription.BuildPickleLambda(ctx, pos)) - .Input(input) - .Build() - .Build() - .Done(); - } - - const auto maybeHopTraits = ExtractHopTraits(aggregate, ctx); - if (!maybeHopTraits) { - return nullptr; - } - const auto hopTraits = maybeHopTraits.Cast(); - - const auto keyLambda = BuildKeySelector(pos, *aggregateInputType, aggregate.Keys().Ptr(), ctx); - const auto timeExtractorLambda = BuildTimeExtractor(hopTraits, ctx); - const auto initLambda = BuildInitHopLambda(aggregate, ctx); - const auto updateLambda = BuildUpdateHopLambda(aggregate, ctx); - const auto saveLambda = BuildSaveHopLambda(aggregate, ctx); - const auto loadLambda = BuildLoadHopLambda(aggregate, ctx); - const auto mergeLambda = BuildMergeHopLambda(aggregate, ctx); - const auto finishLambda = BuildFinishHopLambda(aggregate, ctx); - - const auto streamArg = Build<TCoArgument>(ctx, pos).Name("stream").Done(); - auto multiHoppingCoreBuilder = Build<TCoMultiHoppingCore>(ctx, pos) - .KeyExtractor(keyLambda) - .TimeExtractor(timeExtractorLambda) - .Hop(hopTraits.Hop()) - .Interval(hopTraits.Interval()) - .Delay(hopTraits.Delay()) - .DataWatermarks(hopTraits.DataWatermarks()) - .InitHandler(initLambda) - .UpdateHandler(updateLambda) - .MergeHandler(mergeLambda) - .FinishHandler(finishLambda) - .SaveHandler(saveLambda) - .LoadHandler(loadLambda); - - if (Config->AnalyticsHopping.Get().GetOrElse(false)) { - return Build<TCoPartitionsByKeys>(ctx, node.Pos()) - .Input(input.Ptr()) - .KeySelectorLambda(keyLambda) - .SortDirections<TCoBool>() - .Literal() - .Value("true") - .Build() - .Build() - .SortKeySelectorLambda(timeExtractorLambda) - .ListHandlerLambda() - .Args(streamArg) - .Body<TCoForwardList>() - .Stream(multiHoppingCoreBuilder - .Input<TCoIterator>() - .List(streamArg) - .Build() - .Done()) - .Build() - .Build() - .Done(); - } else { - auto wrappedInput = input.Ptr(); - if (!aggregate.Keys().Empty()) { - // Shuffle input connection by keys - wrappedInput = WrapToShuffle(keysDescription, aggregate, input, ctx); - if (!wrappedInput) { - return nullptr; - } - } - - const auto stage = Build<TDqStage>(ctx, node.Pos()) - .Inputs() - .Add(wrappedInput) - .Build() - .Program() - .Args(streamArg) - .Body<TCoMap>() - .Input(multiHoppingCoreBuilder - .Input<TCoFromFlow>() - .Input(streamArg) - .Build() - .Done()) - .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType)) - .Build() - .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) - .Done(); - - return Build<TDqCnUnionAll>(ctx, node.Pos()) - .Output() - .Stage(stage) - .Index().Build(0) - .Build() - .Done(); - } - } - - TMaybeNode<TCoHoppingTraits> ExtractHopTraits(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - - const auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping"); - if (!hopSetting) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Aggregate over stream must have 'hopping' setting")); - return TMaybeNode<TCoHoppingTraits>(); - } - - const auto maybeTraits = TMaybeNode<TCoHoppingTraits>(hopSetting->Child(1)); - if (!maybeTraits) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Invalid 'hopping' setting in Aggregate")); - return TMaybeNode<TCoHoppingTraits>(); - } - - const auto traits = maybeTraits.Cast(); - - const auto checkIntervalParam = [&] (TExprBase param) -> ui64 { - if (param.Maybe<TCoJust>()) { - param = param.Cast<TCoJust>().Input(); - } - if (!param.Maybe<TCoInterval>()) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Not an interval data ctor")); - return 0; - } - auto value = FromString<i64>(param.Cast<TCoInterval>().Literal().Value()); - if (value <= 0) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval value must be positive")); - return 0; - } - return (ui64)value; - }; - - const auto hop = checkIntervalParam(traits.Hop()); - if (!hop) { - return TMaybeNode<TCoHoppingTraits>(); - } - const auto interval = checkIntervalParam(traits.Interval()); - if (!interval) { - return TMaybeNode<TCoHoppingTraits>(); - } - const auto delay = checkIntervalParam(traits.Delay()); - if (!delay) { - return TMaybeNode<TCoHoppingTraits>(); - } - - if (interval < hop) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval must be greater or equal then hop")); - return TMaybeNode<TCoHoppingTraits>(); - } - if (delay < hop) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Delay must be greater or equal then hop")); - return TMaybeNode<TCoHoppingTraits>(); - } - - return Build<TCoHoppingTraits>(ctx, aggregate.Pos()) - .InitFrom(traits) - .DataWatermarks(Config->AnalyticsHopping.Get().GetOrElse(false) - ? ctx.NewAtom(aggregate.Pos(), "false") - : traits.DataWatermarks().Ptr()) - .Done(); - } - - struct TKeysDescription { - TVector<TString> PickleKeys; - TVector<TString> MemberKeys; - - explicit TKeysDescription(const TStructExprType& rowType, const TCoAtomList& keys) { - for (const auto& key : keys) { - const auto index = rowType.FindItem(key.StringValue()); - Y_ENSURE(index); - - auto itemType = rowType.GetItems()[*index]->GetItemType(); - if (RemoveOptionalType(itemType)->GetKind() == ETypeAnnotationKind::Data) { - MemberKeys.emplace_back(key.StringValue()); - continue; - } - - PickleKeys.emplace_back(key.StringValue()); - } - } - - TExprNode::TPtr BuildPickleLambda(TExprContext& ctx, TPositionHandle pos) const { - TCoArgument arg = Build<TCoArgument>(ctx, pos) - .Name("item") - .Done(); - - TExprBase body = arg; - - for (const auto& key : PickleKeys) { - const auto member = Build<TCoMember>(ctx, pos) - .Name().Build(key) - .Struct(arg) - .Done() - .Ptr(); - - body = Build<TCoReplaceMember>(ctx, pos) - .Struct(body) - .Name().Build(key) - .Item(ctx.NewCallable(pos, "StablePickle", { member })) - .Done(); - } - - return Build<TCoLambda>(ctx, pos) - .Args({arg}) - .Body(body) - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildUnpickleLambda(TExprContext& ctx, TPositionHandle pos, const TStructExprType& rowType) { - TCoArgument arg = Build<TCoArgument>(ctx, pos) - .Name("item") - .Done(); - - TExprBase body = arg; - - for (const auto& key : PickleKeys) { - const auto index = rowType.FindItem(key); - Y_ENSURE(index); - - auto itemType = rowType.GetItems().at(*index)->GetItemType(); - const auto member = Build<TCoMember>(ctx, pos) - .Name().Build(key) - .Struct(arg) - .Done() - .Ptr(); - - body = Build<TCoReplaceMember>(ctx, pos) - .Struct(body) - .Name().Build(key) - .Item(ctx.NewCallable(pos, "Unpickle", { ExpandType(pos, *itemType, ctx), member })) - .Done(); - } - - return Build<TCoLambda>(ctx, pos) - .Args({arg}) - .Body(body) - .Done() - .Ptr(); - } - - TVector<TCoAtom> GetKeysList(TExprContext& ctx, TPositionHandle pos) const { - TVector<TCoAtom> res; - res.reserve(PickleKeys.size() + MemberKeys.size()); - - for (const auto& pickleKey : PickleKeys) { - res.emplace_back(Build<TCoAtom>(ctx, pos).Value(pickleKey).Done()); - } - for (const auto& memberKey : MemberKeys) { - res.emplace_back(Build<TCoAtom>(ctx, pos).Value(memberKey).Done()); - } - return res; - } - - bool NeedPickle() const { - return !PickleKeys.empty(); - } - }; - - TExprNode::TPtr WrapToShuffle( - const TKeysDescription& keysDescription, - const TCoAggregate& aggregate, - const TDqConnection& input, - TExprContext& ctx) - { - auto pos = aggregate.Pos(); - - TDqStageBase mappedInput = input.Output().Stage(); - if (keysDescription.NeedPickle()) { - mappedInput = Build<TDqStage>(ctx, pos) - .Inputs() - .Add<TDqCnMap>() - .Output() - .Stage(input.Output().Stage()) - .Index(input.Output().Index()) - .Build() - .Build() - .Build() - .Program() - .Args({"stream"}) - .Body<TCoMap>() - .Input("stream") - .Lambda(keysDescription.BuildPickleLambda(ctx, pos)) - .Build() - .Build() - .Settings(TDqStageSettings().BuildNode(ctx, pos)) - .Done(); - } - +private: + TMaybeNode<TExprBase> RewriteAsHoppingWindow(const TExprBase node, TExprContext& ctx, const TDqConnection& input) { + const auto aggregate = node.Cast<TCoAggregate>(); + const auto pos = aggregate.Pos(); + + YQL_CLOG(DEBUG, ProviderDq) << "OptimizeStreamingAggregate"; + + EnsureNotDistinct(aggregate); + + const auto aggregateInputType = GetSeqItemType(node.Ptr()->Head().GetTypeAnn())->Cast<TStructExprType>(); + TKeysDescription keysDescription(*aggregateInputType, aggregate.Keys()); + + if (keysDescription.NeedPickle()) { + return Build<TCoMap>(ctx, pos) + .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType)) + .Input<TCoAggregate>() + .InitFrom(aggregate) + .Input<TCoMap>() + .Lambda(keysDescription.BuildPickleLambda(ctx, pos)) + .Input(input) + .Build() + .Build() + .Done(); + } + + const auto maybeHopTraits = ExtractHopTraits(aggregate, ctx); + if (!maybeHopTraits) { + return nullptr; + } + const auto hopTraits = maybeHopTraits.Cast(); + + const auto keyLambda = BuildKeySelector(pos, *aggregateInputType, aggregate.Keys().Ptr(), ctx); + const auto timeExtractorLambda = BuildTimeExtractor(hopTraits, ctx); + const auto initLambda = BuildInitHopLambda(aggregate, ctx); + const auto updateLambda = BuildUpdateHopLambda(aggregate, ctx); + const auto saveLambda = BuildSaveHopLambda(aggregate, ctx); + const auto loadLambda = BuildLoadHopLambda(aggregate, ctx); + const auto mergeLambda = BuildMergeHopLambda(aggregate, ctx); + const auto finishLambda = BuildFinishHopLambda(aggregate, ctx); + + const auto streamArg = Build<TCoArgument>(ctx, pos).Name("stream").Done(); + auto multiHoppingCoreBuilder = Build<TCoMultiHoppingCore>(ctx, pos) + .KeyExtractor(keyLambda) + .TimeExtractor(timeExtractorLambda) + .Hop(hopTraits.Hop()) + .Interval(hopTraits.Interval()) + .Delay(hopTraits.Delay()) + .DataWatermarks(hopTraits.DataWatermarks()) + .InitHandler(initLambda) + .UpdateHandler(updateLambda) + .MergeHandler(mergeLambda) + .FinishHandler(finishLambda) + .SaveHandler(saveLambda) + .LoadHandler(loadLambda); + + if (Config->AnalyticsHopping.Get().GetOrElse(false)) { + return Build<TCoPartitionsByKeys>(ctx, node.Pos()) + .Input(input.Ptr()) + .KeySelectorLambda(keyLambda) + .SortDirections<TCoBool>() + .Literal() + .Value("true") + .Build() + .Build() + .SortKeySelectorLambda(timeExtractorLambda) + .ListHandlerLambda() + .Args(streamArg) + .Body<TCoForwardList>() + .Stream(multiHoppingCoreBuilder + .Input<TCoIterator>() + .List(streamArg) + .Build() + .Done()) + .Build() + .Build() + .Done(); + } else { + auto wrappedInput = input.Ptr(); + if (!aggregate.Keys().Empty()) { + // Shuffle input connection by keys + wrappedInput = WrapToShuffle(keysDescription, aggregate, input, ctx); + if (!wrappedInput) { + return nullptr; + } + } + + const auto stage = Build<TDqStage>(ctx, node.Pos()) + .Inputs() + .Add(wrappedInput) + .Build() + .Program() + .Args(streamArg) + .Body<TCoMap>() + .Input(multiHoppingCoreBuilder + .Input<TCoFromFlow>() + .Input(streamArg) + .Build() + .Done()) + .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType)) + .Build() + .Build() + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Done(); + + return Build<TDqCnUnionAll>(ctx, node.Pos()) + .Output() + .Stage(stage) + .Index().Build(0) + .Build() + .Done(); + } + } + + TMaybeNode<TCoHoppingTraits> ExtractHopTraits(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + + const auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping"); + if (!hopSetting) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Aggregate over stream must have 'hopping' setting")); + return TMaybeNode<TCoHoppingTraits>(); + } + + const auto maybeTraits = TMaybeNode<TCoHoppingTraits>(hopSetting->Child(1)); + if (!maybeTraits) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Invalid 'hopping' setting in Aggregate")); + return TMaybeNode<TCoHoppingTraits>(); + } + + const auto traits = maybeTraits.Cast(); + + const auto checkIntervalParam = [&] (TExprBase param) -> ui64 { + if (param.Maybe<TCoJust>()) { + param = param.Cast<TCoJust>().Input(); + } + if (!param.Maybe<TCoInterval>()) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Not an interval data ctor")); + return 0; + } + auto value = FromString<i64>(param.Cast<TCoInterval>().Literal().Value()); + if (value <= 0) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval value must be positive")); + return 0; + } + return (ui64)value; + }; + + const auto hop = checkIntervalParam(traits.Hop()); + if (!hop) { + return TMaybeNode<TCoHoppingTraits>(); + } + const auto interval = checkIntervalParam(traits.Interval()); + if (!interval) { + return TMaybeNode<TCoHoppingTraits>(); + } + const auto delay = checkIntervalParam(traits.Delay()); + if (!delay) { + return TMaybeNode<TCoHoppingTraits>(); + } + + if (interval < hop) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval must be greater or equal then hop")); + return TMaybeNode<TCoHoppingTraits>(); + } + if (delay < hop) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Delay must be greater or equal then hop")); + return TMaybeNode<TCoHoppingTraits>(); + } + + return Build<TCoHoppingTraits>(ctx, aggregate.Pos()) + .InitFrom(traits) + .DataWatermarks(Config->AnalyticsHopping.Get().GetOrElse(false) + ? ctx.NewAtom(aggregate.Pos(), "false") + : traits.DataWatermarks().Ptr()) + .Done(); + } + + struct TKeysDescription { + TVector<TString> PickleKeys; + TVector<TString> MemberKeys; + + explicit TKeysDescription(const TStructExprType& rowType, const TCoAtomList& keys) { + for (const auto& key : keys) { + const auto index = rowType.FindItem(key.StringValue()); + Y_ENSURE(index); + + auto itemType = rowType.GetItems()[*index]->GetItemType(); + if (RemoveOptionalType(itemType)->GetKind() == ETypeAnnotationKind::Data) { + MemberKeys.emplace_back(key.StringValue()); + continue; + } + + PickleKeys.emplace_back(key.StringValue()); + } + } + + TExprNode::TPtr BuildPickleLambda(TExprContext& ctx, TPositionHandle pos) const { + TCoArgument arg = Build<TCoArgument>(ctx, pos) + .Name("item") + .Done(); + + TExprBase body = arg; + + for (const auto& key : PickleKeys) { + const auto member = Build<TCoMember>(ctx, pos) + .Name().Build(key) + .Struct(arg) + .Done() + .Ptr(); + + body = Build<TCoReplaceMember>(ctx, pos) + .Struct(body) + .Name().Build(key) + .Item(ctx.NewCallable(pos, "StablePickle", { member })) + .Done(); + } + + return Build<TCoLambda>(ctx, pos) + .Args({arg}) + .Body(body) + .Done() + .Ptr(); + } + + TExprNode::TPtr BuildUnpickleLambda(TExprContext& ctx, TPositionHandle pos, const TStructExprType& rowType) { + TCoArgument arg = Build<TCoArgument>(ctx, pos) + .Name("item") + .Done(); + + TExprBase body = arg; + + for (const auto& key : PickleKeys) { + const auto index = rowType.FindItem(key); + Y_ENSURE(index); + + auto itemType = rowType.GetItems().at(*index)->GetItemType(); + const auto member = Build<TCoMember>(ctx, pos) + .Name().Build(key) + .Struct(arg) + .Done() + .Ptr(); + + body = Build<TCoReplaceMember>(ctx, pos) + .Struct(body) + .Name().Build(key) + .Item(ctx.NewCallable(pos, "Unpickle", { ExpandType(pos, *itemType, ctx), member })) + .Done(); + } + + return Build<TCoLambda>(ctx, pos) + .Args({arg}) + .Body(body) + .Done() + .Ptr(); + } + + TVector<TCoAtom> GetKeysList(TExprContext& ctx, TPositionHandle pos) const { + TVector<TCoAtom> res; + res.reserve(PickleKeys.size() + MemberKeys.size()); + + for (const auto& pickleKey : PickleKeys) { + res.emplace_back(Build<TCoAtom>(ctx, pos).Value(pickleKey).Done()); + } + for (const auto& memberKey : MemberKeys) { + res.emplace_back(Build<TCoAtom>(ctx, pos).Value(memberKey).Done()); + } + return res; + } + + bool NeedPickle() const { + return !PickleKeys.empty(); + } + }; + + TExprNode::TPtr WrapToShuffle( + const TKeysDescription& keysDescription, + const TCoAggregate& aggregate, + const TDqConnection& input, + TExprContext& ctx) + { + auto pos = aggregate.Pos(); + + TDqStageBase mappedInput = input.Output().Stage(); + if (keysDescription.NeedPickle()) { + mappedInput = Build<TDqStage>(ctx, pos) + .Inputs() + .Add<TDqCnMap>() + .Output() + .Stage(input.Output().Stage()) + .Index(input.Output().Index()) + .Build() + .Build() + .Build() + .Program() + .Args({"stream"}) + .Body<TCoMap>() + .Input("stream") + .Lambda(keysDescription.BuildPickleLambda(ctx, pos)) + .Build() + .Build() + .Settings(TDqStageSettings().BuildNode(ctx, pos)) + .Done(); + } + return Build<TDqCnHashShuffle>(ctx, pos) - .Output() - .Stage(mappedInput) - .Index().Value("0").Build() - .Build() - .KeyColumns() - .Add(keysDescription.GetKeysList(ctx, pos)) - .Build() - .Done() - .Ptr(); - } - - void EnsureNotDistinct(const TCoAggregate& aggregate) { - const auto& aggregateHandlers = aggregate.Handlers(); - - YQL_ENSURE( - AllOf(aggregateHandlers, [](const auto& t){ return !t.DistinctName(); }), - "Distinct is not supported for aggregation with hop"); - } - - TExprNode::TPtr BuildTimeExtractor(const TCoHoppingTraits& hoppingTraits, TExprContext& ctx) { - const auto pos = hoppingTraits.Pos(); - - if (hoppingTraits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>()->GetSize() == 0) { - // The case when no fields are used in lambda. F.e. when it has only DependsOn. - return ctx.DeepCopyLambda(hoppingTraits.TimeExtractor().Ref()); - } - - return Build<TCoLambda>(ctx, pos) - .Args({"item"}) - .Body<TExprApplier>() - .Apply(hoppingTraits.TimeExtractor()) - .With<TCoSafeCast>(0) - .Type(hoppingTraits.ItemType()) - .Value("item") - .Build() - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildInitHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto& aggregateHandlers = aggregate.Handlers(); - - const auto initItemArg = Build<TCoArgument>(ctx, pos).Name("item").Done(); - - TVector<TExprBase> structItems; - structItems.reserve(aggregateHandlers.Size()); - - ui32 index = 0; - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast<TCoAggregateTuple>(); - - TMaybeNode<TExprBase> applier; - if (tuple.Trait().InitHandler().Args().Size() == 1) { - applier = Build<TExprApplier>(ctx, pos) - .Apply(tuple.Trait().InitHandler()) - .With(0, initItemArg) - .Done(); - } else { - applier = Build<TExprApplier>(ctx, pos) - .Apply(tuple.Trait().InitHandler()) - .With(0, initItemArg) - .With<TCoUint32>(1) - .Literal().Build(ToString(index)) - .Build() - .Done(); - } - - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name().Build(BuildColumnName(tuple.ColumnName())) - .Value(applier) - .Done()); - ++index; - } - - return Build<TCoLambda>(ctx, pos) - .Args({initItemArg}) - .Body<TCoAsStruct>() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildUpdateHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto aggregateHandlers = aggregate.Handlers(); - - const auto updateItemArg = Build<TCoArgument>(ctx, pos).Name("item").Done(); - const auto updateStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done(); - - TVector<TExprBase> structItems; - structItems.reserve(aggregateHandlers.Size()); - - i32 index = 0; - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast<TCoAggregateTuple>(); - const TString columnName = BuildColumnName(tuple.ColumnName()); - - const auto member = Build<TCoMember>(ctx, pos) - .Struct(updateStateArg) - .Name().Build(columnName) - .Done(); - - TMaybeNode<TExprBase> applier; - if (tuple.Trait().UpdateHandler().Args().Size() == 2) { - applier = Build<TExprApplier>(ctx, pos) - .Apply(tuple.Trait().UpdateHandler()) - .With(0, updateItemArg) - .With(1, member) - .Done(); - } else { - applier = Build<TExprApplier>(ctx, pos) - .Apply(tuple.Trait().UpdateHandler()) - .With(0, updateItemArg) - .With(1, member) - .With<TCoUint32>(2) - .Literal().Build(ToString(index)) - .Build() - .Done(); - } - - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name().Build(columnName) - .Value(applier) - .Done()); - ++index; - } - - return Build<TCoLambda>(ctx, pos) - .Args({updateItemArg, updateStateArg}) - .Body<TCoAsStruct>() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildMergeHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto& aggregateHandlers = aggregate.Handlers(); - - const auto mergeState1Arg = Build<TCoArgument>(ctx, pos).Name("state1").Done(); - const auto mergeState2Arg = Build<TCoArgument>(ctx, pos).Name("state2").Done(); - - TVector<TExprBase> structItems; - structItems.reserve(aggregateHandlers.Size()); - - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast<TCoAggregateTuple>(); - const TString columnName = BuildColumnName(tuple.ColumnName()); - - const auto member1 = Build<TCoMember>(ctx, pos) - .Struct(mergeState1Arg) - .Name().Build(columnName) - .Done(); - const auto member2 = Build<TCoMember>(ctx, pos) - .Struct(mergeState2Arg) - .Name().Build(columnName) - .Done(); - - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name().Build(columnName) - .Value<TExprApplier>() - .Apply(tuple.Trait().MergeHandler()) - .With(0, member1) - .With(1, member2) - .Build() - .Done()); - } - - return Build<TCoLambda>(ctx, pos) - .Args({mergeState1Arg, mergeState2Arg}) - .Body<TCoAsStruct>() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildFinishHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto keyColumns = aggregate.Keys(); - const auto aggregateHandlers = aggregate.Handlers(); - - const auto finishKeyArg = Build<TCoArgument>(ctx, pos).Name("key").Done(); - const auto finishStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done(); - const auto finishTimeArg = Build<TCoArgument>(ctx, pos).Name("time").Done(); - - TVector<TExprBase> structItems; - structItems.reserve(keyColumns.Size() + aggregateHandlers.Size() + 1); - - if (keyColumns.Size() == 1) { - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name(keyColumns.Item(0)) - .Value(finishKeyArg) - .Done()); - } else { - for (size_t i = 0; i < keyColumns.Size(); ++i) { - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name(keyColumns.Item(i)) - .Value<TCoNth>() - .Tuple(finishKeyArg) - .Index<TCoAtom>() - .Value(ToString(i)) - .Build() - .Build() - .Done()); - } - } - - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast<TCoAggregateTuple>(); - const TString compoundColumnName = BuildColumnName(tuple.ColumnName()); - - const auto member = Build<TCoMember>(ctx, pos) - .Struct(finishStateArg) - .Name().Build(compoundColumnName) - .Done(); - - if (tuple.ColumnName().Maybe<TCoAtom>()) { - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name().Build(compoundColumnName) - .Value<TExprApplier>() - .Apply(tuple.Trait().FinishHandler()) - .With(0, member) - .Build() - .Done()); - - continue; - } - - if (const auto namesList = tuple.ColumnName().Maybe<TCoAtomList>()) { - const auto expApplier = Build<TExprApplier>(ctx, pos) - .Apply(tuple.Trait().FinishHandler()) - .With(0, member) - .Done(); - - int index = 0; - for (const auto columnName : namesList.Cast()) { - const auto extracter = Build<TCoNth>(ctx, pos) - .Tuple(expApplier) - .Index<TCoAtom>().Build(index++) - .Done(); - - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name(columnName) - .Value(extracter) - .Done()); - } - - continue; - } - - YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: " - << tuple.ColumnName().Ptr()->Dump()); - } - - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name().Build(YQL_TIME) - .Value(finishTimeArg) - .Done()); - - return Build<TCoLambda>(ctx, pos) - .Args({finishKeyArg, finishStateArg, finishTimeArg}) - .Body<TCoAsStruct>() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildSaveHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto aggregateHandlers = aggregate.Handlers(); - - const auto saveStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done(); - - TVector<TExprBase> structItems; - structItems.reserve(aggregateHandlers.Size()); - - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast<TCoAggregateTuple>(); - const TString columnName = BuildColumnName(tuple.ColumnName()); - - const auto member = Build<TCoMember>(ctx, pos) - .Struct(saveStateArg) - .Name().Build(columnName) - .Done(); - - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name().Build(columnName) - .Value<TExprApplier>() - .Apply(tuple.Trait().SaveHandler()) - .With(0, member) - .Build() - .Done()); - } - - return Build<TCoLambda>(ctx, pos) - .Args({saveStateArg}) - .Body<TCoAsStruct>() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildLoadHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto aggregateHandlers = aggregate.Handlers(); - - TCoArgument loadStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done(); - - TVector<TExprBase> structItems; - structItems.reserve(aggregateHandlers.Size()); - - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast<TCoAggregateTuple>(); - const TString columnName = BuildColumnName(tuple.ColumnName()); - - const auto member = Build<TCoMember>(ctx, pos) - .Struct(loadStateArg) - .Name().Build(columnName) - .Done(); - - structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name().Build(columnName) - .Value<TExprApplier>() - .Apply(tuple.Trait().LoadHandler()) - .With(0, member) - .Build() - .Done()); - } - - return Build<TCoLambda>(ctx, pos) - .Args({loadStateArg}) - .Body<TCoAsStruct>() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - -private: - TDqConfiguration::TPtr Config; + .Output() + .Stage(mappedInput) + .Index().Value("0").Build() + .Build() + .KeyColumns() + .Add(keysDescription.GetKeysList(ctx, pos)) + .Build() + .Done() + .Ptr(); + } + + void EnsureNotDistinct(const TCoAggregate& aggregate) { + const auto& aggregateHandlers = aggregate.Handlers(); + + YQL_ENSURE( + AllOf(aggregateHandlers, [](const auto& t){ return !t.DistinctName(); }), + "Distinct is not supported for aggregation with hop"); + } + + TExprNode::TPtr BuildTimeExtractor(const TCoHoppingTraits& hoppingTraits, TExprContext& ctx) { + const auto pos = hoppingTraits.Pos(); + + if (hoppingTraits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>()->GetSize() == 0) { + // The case when no fields are used in lambda. F.e. when it has only DependsOn. + return ctx.DeepCopyLambda(hoppingTraits.TimeExtractor().Ref()); + } + + return Build<TCoLambda>(ctx, pos) + .Args({"item"}) + .Body<TExprApplier>() + .Apply(hoppingTraits.TimeExtractor()) + .With<TCoSafeCast>(0) + .Type(hoppingTraits.ItemType()) + .Value("item") + .Build() + .Build() + .Done() + .Ptr(); + } + + TExprNode::TPtr BuildInitHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto& aggregateHandlers = aggregate.Handlers(); + + const auto initItemArg = Build<TCoArgument>(ctx, pos).Name("item").Done(); + + TVector<TExprBase> structItems; + structItems.reserve(aggregateHandlers.Size()); + + ui32 index = 0; + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast<TCoAggregateTuple>(); + + TMaybeNode<TExprBase> applier; + if (tuple.Trait().InitHandler().Args().Size() == 1) { + applier = Build<TExprApplier>(ctx, pos) + .Apply(tuple.Trait().InitHandler()) + .With(0, initItemArg) + .Done(); + } else { + applier = Build<TExprApplier>(ctx, pos) + .Apply(tuple.Trait().InitHandler()) + .With(0, initItemArg) + .With<TCoUint32>(1) + .Literal().Build(ToString(index)) + .Build() + .Done(); + } + + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(BuildColumnName(tuple.ColumnName())) + .Value(applier) + .Done()); + ++index; + } + + return Build<TCoLambda>(ctx, pos) + .Args({initItemArg}) + .Body<TCoAsStruct>() + .Add(structItems) + .Build() + .Done() + .Ptr(); + } + + TExprNode::TPtr BuildUpdateHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto aggregateHandlers = aggregate.Handlers(); + + const auto updateItemArg = Build<TCoArgument>(ctx, pos).Name("item").Done(); + const auto updateStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done(); + + TVector<TExprBase> structItems; + structItems.reserve(aggregateHandlers.Size()); + + i32 index = 0; + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast<TCoAggregateTuple>(); + const TString columnName = BuildColumnName(tuple.ColumnName()); + + const auto member = Build<TCoMember>(ctx, pos) + .Struct(updateStateArg) + .Name().Build(columnName) + .Done(); + + TMaybeNode<TExprBase> applier; + if (tuple.Trait().UpdateHandler().Args().Size() == 2) { + applier = Build<TExprApplier>(ctx, pos) + .Apply(tuple.Trait().UpdateHandler()) + .With(0, updateItemArg) + .With(1, member) + .Done(); + } else { + applier = Build<TExprApplier>(ctx, pos) + .Apply(tuple.Trait().UpdateHandler()) + .With(0, updateItemArg) + .With(1, member) + .With<TCoUint32>(2) + .Literal().Build(ToString(index)) + .Build() + .Done(); + } + + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(columnName) + .Value(applier) + .Done()); + ++index; + } + + return Build<TCoLambda>(ctx, pos) + .Args({updateItemArg, updateStateArg}) + .Body<TCoAsStruct>() + .Add(structItems) + .Build() + .Done() + .Ptr(); + } + + TExprNode::TPtr BuildMergeHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto& aggregateHandlers = aggregate.Handlers(); + + const auto mergeState1Arg = Build<TCoArgument>(ctx, pos).Name("state1").Done(); + const auto mergeState2Arg = Build<TCoArgument>(ctx, pos).Name("state2").Done(); + + TVector<TExprBase> structItems; + structItems.reserve(aggregateHandlers.Size()); + + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast<TCoAggregateTuple>(); + const TString columnName = BuildColumnName(tuple.ColumnName()); + + const auto member1 = Build<TCoMember>(ctx, pos) + .Struct(mergeState1Arg) + .Name().Build(columnName) + .Done(); + const auto member2 = Build<TCoMember>(ctx, pos) + .Struct(mergeState2Arg) + .Name().Build(columnName) + .Done(); + + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(columnName) + .Value<TExprApplier>() + .Apply(tuple.Trait().MergeHandler()) + .With(0, member1) + .With(1, member2) + .Build() + .Done()); + } + + return Build<TCoLambda>(ctx, pos) + .Args({mergeState1Arg, mergeState2Arg}) + .Body<TCoAsStruct>() + .Add(structItems) + .Build() + .Done() + .Ptr(); + } + + TExprNode::TPtr BuildFinishHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto keyColumns = aggregate.Keys(); + const auto aggregateHandlers = aggregate.Handlers(); + + const auto finishKeyArg = Build<TCoArgument>(ctx, pos).Name("key").Done(); + const auto finishStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done(); + const auto finishTimeArg = Build<TCoArgument>(ctx, pos).Name("time").Done(); + + TVector<TExprBase> structItems; + structItems.reserve(keyColumns.Size() + aggregateHandlers.Size() + 1); + + if (keyColumns.Size() == 1) { + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name(keyColumns.Item(0)) + .Value(finishKeyArg) + .Done()); + } else { + for (size_t i = 0; i < keyColumns.Size(); ++i) { + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name(keyColumns.Item(i)) + .Value<TCoNth>() + .Tuple(finishKeyArg) + .Index<TCoAtom>() + .Value(ToString(i)) + .Build() + .Build() + .Done()); + } + } + + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast<TCoAggregateTuple>(); + const TString compoundColumnName = BuildColumnName(tuple.ColumnName()); + + const auto member = Build<TCoMember>(ctx, pos) + .Struct(finishStateArg) + .Name().Build(compoundColumnName) + .Done(); + + if (tuple.ColumnName().Maybe<TCoAtom>()) { + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(compoundColumnName) + .Value<TExprApplier>() + .Apply(tuple.Trait().FinishHandler()) + .With(0, member) + .Build() + .Done()); + + continue; + } + + if (const auto namesList = tuple.ColumnName().Maybe<TCoAtomList>()) { + const auto expApplier = Build<TExprApplier>(ctx, pos) + .Apply(tuple.Trait().FinishHandler()) + .With(0, member) + .Done(); + + int index = 0; + for (const auto columnName : namesList.Cast()) { + const auto extracter = Build<TCoNth>(ctx, pos) + .Tuple(expApplier) + .Index<TCoAtom>().Build(index++) + .Done(); + + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name(columnName) + .Value(extracter) + .Done()); + } + + continue; + } + + YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: " + << tuple.ColumnName().Ptr()->Dump()); + } + + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(YQL_TIME) + .Value(finishTimeArg) + .Done()); + + return Build<TCoLambda>(ctx, pos) + .Args({finishKeyArg, finishStateArg, finishTimeArg}) + .Body<TCoAsStruct>() + .Add(structItems) + .Build() + .Done() + .Ptr(); + } + + TExprNode::TPtr BuildSaveHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto aggregateHandlers = aggregate.Handlers(); + + const auto saveStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done(); + + TVector<TExprBase> structItems; + structItems.reserve(aggregateHandlers.Size()); + + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast<TCoAggregateTuple>(); + const TString columnName = BuildColumnName(tuple.ColumnName()); + + const auto member = Build<TCoMember>(ctx, pos) + .Struct(saveStateArg) + .Name().Build(columnName) + .Done(); + + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(columnName) + .Value<TExprApplier>() + .Apply(tuple.Trait().SaveHandler()) + .With(0, member) + .Build() + .Done()); + } + + return Build<TCoLambda>(ctx, pos) + .Args({saveStateArg}) + .Body<TCoAsStruct>() + .Add(structItems) + .Build() + .Done() + .Ptr(); + } + + TExprNode::TPtr BuildLoadHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto aggregateHandlers = aggregate.Handlers(); + + TCoArgument loadStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done(); + + TVector<TExprBase> structItems; + structItems.reserve(aggregateHandlers.Size()); + + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast<TCoAggregateTuple>(); + const TString columnName = BuildColumnName(tuple.ColumnName()); + + const auto member = Build<TCoMember>(ctx, pos) + .Struct(loadStateArg) + .Name().Build(columnName) + .Done(); + + structItems.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(columnName) + .Value<TExprApplier>() + .Apply(tuple.Trait().LoadHandler()) + .With(0, member) + .Build() + .Done()); + } + + return Build<TCoLambda>(ctx, pos) + .Args({loadStateArg}) + .Body<TCoAsStruct>() + .Add(structItems) + .Build() + .Done() + .Ptr(); + } + +private: + TDqConfiguration::TPtr Config; }; -THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config) { - return THolder(new TDqsLogicalOptProposalTransformer(typeCtx, config)); +THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config) { + return THolder(new TDqsLogicalOptProposalTransformer(typeCtx, config)); } } // NYql::NDqs diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.h b/ydb/library/yql/providers/dq/opt/logical_optimize.h index 13eb9aff9d8..54352ac344e 100644 --- a/ydb/library/yql/providers/dq/opt/logical_optimize.h +++ b/ydb/library/yql/providers/dq/opt/logical_optimize.h @@ -8,6 +8,6 @@ namespace NYql::NDqs { -THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config); +THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config); } // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 42ae97ad833..8759f442219 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -647,9 +647,9 @@ private: TUploadList uploadList; auto lambdaResult = GetLambda(&lambda, &untrustedUdfFlag, &level, &uploadList, result, ctx, hasGraphParams); if (lambdaResult.first.Level == TStatus::Error) { - if (State->Settings->FallbackPolicy.Get().GetOrElse("default") == "never" || State->TypeCtx->ForceDq) { - return SyncError(); - } + if (State->Settings->FallbackPolicy.Get().GetOrElse("default") == "never" || State->TypeCtx->ForceDq) { + return SyncError(); + } return Fallback(); } if (lambdaResult.first.Level != TStatus::Ok) { diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp index e8eb1ece76c..a8b5e598a08 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp @@ -27,7 +27,7 @@ class TDqDataProviderSink: public TDataProviderBase { public: TDqDataProviderSink(const TDqStatePtr& state) : State(state) - , LogOptTransformer([state] () { return CreateDqsLogOptTransformer(/*TODO: State->TypeCtx);*/nullptr, state->Settings); }) + , LogOptTransformer([state] () { return CreateDqsLogOptTransformer(/*TODO: State->TypeCtx);*/nullptr, state->Settings); }) , PhyOptTransformer([] () { return CreateDqsPhyOptTransformer(/*TODO: State->TypeCtx*/nullptr); }) , PhysicalFinalizingTransformer([] () { return CreateDqsFinalizingOptTransformer(); }) , TypeAnnotationTransformer([state] () { return CreateDqsDataSinkTypeAnnotationTransformer(state->TypeCtx); }) diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp index 7b85f56f93f..e26956545e9 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp @@ -1,134 +1,134 @@ -#include "dq_pq_read_actor.h" -#include "probes.h" - +#include "dq_pq_read_actor.h" +#include "probes.h" + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> #include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> #include <ydb/library/yql/dq/common/dq_common.h> #include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h> - + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h> #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/providers/pq/proto/dq_io_state.pb.h> #include <ydb/library/yql/utils/yql_panic.h> - + #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> -#include <library/cpp/lwtrace/mon/mon_lwtrace.h> - -#include <util/generic/algorithm.h> -#include <util/generic/hash.h> -#include <util/generic/utility.h> - -#include <queue> -#include <variant> - -namespace NKikimrServices { +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> +#include <library/cpp/lwtrace/mon/mon_lwtrace.h> + +#include <util/generic/algorithm.h> +#include <util/generic/hash.h> +#include <util/generic/utility.h> + +#include <queue> +#include <variant> + +namespace NKikimrServices { // using constant value from ydb/core/protos/services.proto // but to avoid peerdir on ydb/core/protos we introduce this constant - constexpr ui32 KQP_COMPUTE = 535; -}; - -const TString LogPrefix = "PQ sink. "; - -#define SINK_LOG_T(s) \ - LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_D(s) \ - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_I(s) \ - LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_W(s) \ - LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_N(s) \ - LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_E(s) \ - LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_C(s) \ - LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG(prio, s) \ - LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, LogPrefix << s) - -namespace NYql::NDq { - -using namespace NActors; -using namespace NLog; -using namespace NKikimr::NMiniKQL; - -constexpr ui32 StateVersion = 1; - -namespace { - -LWTRACE_USING(DQ_PQ_PROVIDER); - -struct TEvPrivate { - // Event ids - enum EEv : ui32 { - EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), - - EvSourceDataReady = EvBegin, - - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)"); - - // Events - - struct TEvSourceDataReady : public TEventLocal<TEvSourceDataReady, EvSourceDataReady> {}; -}; - -} // namespace - -class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqSourceActor { -public: - using TPartitionKey = std::pair<TString, ui64>; // Cluster, partition id. - - TDqPqReadActor( - ui64 inputIndex, - const TString& txId, - const THolderFactory& holderFactory, - NPq::NProto::TDqPqTopicSource&& sourceParams, - NPq::NProto::TDqReadTaskParams&& readParams, + constexpr ui32 KQP_COMPUTE = 535; +}; + +const TString LogPrefix = "PQ sink. "; + +#define SINK_LOG_T(s) \ + LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_D(s) \ + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_I(s) \ + LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_W(s) \ + LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_N(s) \ + LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_E(s) \ + LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_C(s) \ + LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG(prio, s) \ + LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, LogPrefix << s) + +namespace NYql::NDq { + +using namespace NActors; +using namespace NLog; +using namespace NKikimr::NMiniKQL; + +constexpr ui32 StateVersion = 1; + +namespace { + +LWTRACE_USING(DQ_PQ_PROVIDER); + +struct TEvPrivate { + // Event ids + enum EEv : ui32 { + EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), + + EvSourceDataReady = EvBegin, + + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)"); + + // Events + + struct TEvSourceDataReady : public TEventLocal<TEvSourceDataReady, EvSourceDataReady> {}; +}; + +} // namespace + +class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqSourceActor { +public: + using TPartitionKey = std::pair<TString, ui64>; // Cluster, partition id. + + TDqPqReadActor( + ui64 inputIndex, + const TString& txId, + const THolderFactory& holderFactory, + NPq::NProto::TDqPqTopicSource&& sourceParams, + NPq::NProto::TDqReadTaskParams&& readParams, NYdb::TDriver driver, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, - ICallbacks* callbacks, - i64 bufferSize) - : TActor<TDqPqReadActor>(&TDqPqReadActor::StateFunc) - , InputIndex(inputIndex) - , TxId(txId) + ICallbacks* callbacks, + i64 bufferSize) + : TActor<TDqPqReadActor>(&TDqPqReadActor::StateFunc) + , InputIndex(inputIndex) + , TxId(txId) , BufferSize(bufferSize) - , HolderFactory(holderFactory) - , Driver(std::move(driver)) + , HolderFactory(holderFactory) + , Driver(std::move(driver)) , CredentialsProviderFactory(std::move(credentialsProviderFactory)) - , SourceParams(std::move(sourceParams)) - , ReadParams(std::move(readParams)) + , SourceParams(std::move(sourceParams)) + , ReadParams(std::move(readParams)) , StartingMessageTimestamp(TInstant::Now()) - , Callbacks(callbacks) - { + , Callbacks(callbacks) + { Y_UNUSED(HolderFactory); } NYdb::NPersQueue::TPersQueueClientSettings GetPersQueueClientSettings() const { - NYdb::NPersQueue::TPersQueueClientSettings opts; - opts.Database(SourceParams.GetDatabase()) - .DiscoveryEndpoint(SourceParams.GetEndpoint()) - .EnableSsl(SourceParams.GetUseSsl()) + NYdb::NPersQueue::TPersQueueClientSettings opts; + opts.Database(SourceParams.GetDatabase()) + .DiscoveryEndpoint(SourceParams.GetEndpoint()) + .EnableSsl(SourceParams.GetUseSsl()) .CredentialsProviderFactory(CredentialsProviderFactory); - + return opts; - } - + } + static constexpr char ActorName[] = "DQ_PQ_READ_ACTOR"; -public: +public: void SaveState(const NDqProto::TCheckpoint& checkpoint, NDqProto::TSourceState& state) override { NPq::NProto::TDqPqTopicSourceState stateProto; @@ -138,15 +138,15 @@ public: topic->SetDatabase(SourceParams.GetDatabase()); topic->SetTopicPath(SourceParams.GetTopicPath()); - for (const auto& [clusterAndPartition, offset] : PartitionToOffset) { - const auto& [cluster, partition] = clusterAndPartition; + for (const auto& [clusterAndPartition, offset] : PartitionToOffset) { + const auto& [cluster, partition] = clusterAndPartition; NPq::NProto::TDqPqTopicSourceState::TPartitionReadState* partitionState = stateProto.AddPartitions(); partitionState->SetTopicIndex(0); // Now we are supporting only one topic per source. partitionState->SetCluster(cluster); partitionState->SetPartition(partition); partitionState->SetOffset(offset); - } - + } + stateProto.SetStartingMessageTimestampMs(StartingMessageTimestamp.MilliSeconds()); TString stateBlob; @@ -156,10 +156,10 @@ public: data->SetVersion(StateVersion); data->SetBlob(stateBlob); - DeferredCommits.emplace(checkpoint.GetId(), std::move(CurrentDeferredCommit)); - CurrentDeferredCommit = NYdb::NPersQueue::TDeferredCommit(); - } - + DeferredCommits.emplace(checkpoint.GetId(), std::move(CurrentDeferredCommit)); + CurrentDeferredCommit = NYdb::NPersQueue::TDeferredCommit(); + } + void LoadState(const NDqProto::TSourceState& state) override { TInstant minStartingMessageTs = state.DataSize() ? TInstant::Max() : StartingMessageTimestamp; for (const auto& stateData : state.GetData()) { @@ -187,20 +187,20 @@ public: ReadSession.reset(); GetReadSession(); } - } - + } + void CommitState(const NDqProto::TCheckpoint& checkpoint) override { - const auto checkpointId = checkpoint.GetId(); - while (!DeferredCommits.empty() && DeferredCommits.front().first <= checkpointId) { - DeferredCommits.front().second.Commit(); - DeferredCommits.pop(); - } - } - + const auto checkpointId = checkpoint.GetId(); + while (!DeferredCommits.empty() && DeferredCommits.front().first <= checkpointId) { + DeferredCommits.front().second.Commit(); + DeferredCommits.pop(); + } + } + ui64 GetInputIndex() const override { - return InputIndex; - }; - + return InputIndex; + }; + NYdb::NPersQueue::TPersQueueClient& GetPersQueueClient() { if (!PersQueueClient) { PersQueueClient = std::make_unique<NYdb::NPersQueue::TPersQueueClient>(Driver, GetPersQueueClientSettings()); @@ -215,84 +215,84 @@ public: return *ReadSession; } -private: - STRICT_STFUNC(StateFunc, - hFunc(TEvPrivate::TEvSourceDataReady, Handle); - ) - - void Handle(TEvPrivate::TEvSourceDataReady::TPtr& ev) { +private: + STRICT_STFUNC(StateFunc, + hFunc(TEvPrivate::TEvSourceDataReady, Handle); + ) + + void Handle(TEvPrivate::TEvSourceDataReady::TPtr& ev) { SubscribedOnEvent = false; - Y_UNUSED(ev); - Callbacks->OnNewSourceDataArrived(InputIndex); - } - + Y_UNUSED(ev); + Callbacks->OnNewSourceDataArrived(InputIndex); + } + // IActor & IDqSourceActor void PassAway() override { // Is called from Compute Actor if (ReadSession) { ReadSession->Close(TDuration::Zero()); ReadSession.reset(); } - PersQueueClient.reset(); + PersQueueClient.reset(); TActor<TDqPqReadActor>::PassAway(); - } - + } + i64 GetSourceData(NKikimr::NMiniKQL::TUnboxedValueVector& buffer, bool&, i64 freeSpace) override { auto events = GetReadSession().GetEvents(false, TMaybe<size_t>(), static_cast<size_t>(Max<i64>(freeSpace, 0))); - - ui32 batchSize = 0; - for (auto& event : events) { - if (const auto* val = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent>(&event)) { - batchSize += val->GetMessages().size(); - } - } - buffer.clear(); - buffer.reserve(batchSize); - - i64 usedSpace = 0; - for (auto& event : events) { - std::visit(TPQEventProcessor{*this, buffer, usedSpace}, event); - } - - SubscribeOnNextEvent(); - - return usedSpace; - } - -private: + + ui32 batchSize = 0; + for (auto& event : events) { + if (const auto* val = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent>(&event)) { + batchSize += val->GetMessages().size(); + } + } + buffer.clear(); + buffer.reserve(batchSize); + + i64 usedSpace = 0; + for (auto& event : events) { + std::visit(TPQEventProcessor{*this, buffer, usedSpace}, event); + } + + SubscribeOnNextEvent(); + + return usedSpace; + } + +private: NYdb::NPersQueue::TReadSessionSettings GetReadSessionSettings() const { - NYdb::NPersQueue::TTopicReadSettings topicReadSettings; - topicReadSettings.Path(SourceParams.GetTopicPath()); - ui64 currentPartition = ReadParams.GetPartitioningParams().GetEachTopicPartitionGroupId(); - do { - topicReadSettings.AppendPartitionGroupIds(currentPartition + 1); // 1-based. - currentPartition += ReadParams.GetPartitioningParams().GetDqPartitionsCount(); - } while (currentPartition < ReadParams.GetPartitioningParams().GetTopicPartitionsCount()); - - return NYdb::NPersQueue::TReadSessionSettings() - .DisableClusterDiscovery(SourceParams.GetClusterType() == NPq::NProto::DataStreams) - .AppendTopics(topicReadSettings) - .ConsumerName(SourceParams.GetConsumerName()) + NYdb::NPersQueue::TTopicReadSettings topicReadSettings; + topicReadSettings.Path(SourceParams.GetTopicPath()); + ui64 currentPartition = ReadParams.GetPartitioningParams().GetEachTopicPartitionGroupId(); + do { + topicReadSettings.AppendPartitionGroupIds(currentPartition + 1); // 1-based. + currentPartition += ReadParams.GetPartitioningParams().GetDqPartitionsCount(); + } while (currentPartition < ReadParams.GetPartitioningParams().GetTopicPartitionsCount()); + + return NYdb::NPersQueue::TReadSessionSettings() + .DisableClusterDiscovery(SourceParams.GetClusterType() == NPq::NProto::DataStreams) + .AppendTopics(topicReadSettings) + .ConsumerName(SourceParams.GetConsumerName()) .MaxMemoryUsageBytes(BufferSize) .StartingMessageTimestamp(StartingMessageTimestamp); - } - - void UpdateStateWithNewReadData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& event) { - if (event.GetMessages().empty()) { - return; - } - - assert(MaxElementBy(event.GetMessages(), [](const auto& message){ return message.GetOffset(); }) - ->GetOffset() == event.GetMessages().back().GetOffset()); - - const auto maxOffset = event.GetMessages().back().GetOffset(); - PartitionToOffset[MakePartitionKey(event.GetPartitionStream())] = maxOffset + 1; // Next offset to read from. - } - - static TPartitionKey MakePartitionKey(const NYdb::NPersQueue::TPartitionStream::TPtr& partitionStreamPtr) { - return std::make_pair(partitionStreamPtr->GetCluster(), partitionStreamPtr->GetPartitionId()); - } - - void SubscribeOnNextEvent() { + } + + void UpdateStateWithNewReadData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& event) { + if (event.GetMessages().empty()) { + return; + } + + assert(MaxElementBy(event.GetMessages(), [](const auto& message){ return message.GetOffset(); }) + ->GetOffset() == event.GetMessages().back().GetOffset()); + + const auto maxOffset = event.GetMessages().back().GetOffset(); + PartitionToOffset[MakePartitionKey(event.GetPartitionStream())] = maxOffset + 1; // Next offset to read from. + } + + static TPartitionKey MakePartitionKey(const NYdb::NPersQueue::TPartitionStream::TPtr& partitionStreamPtr) { + return std::make_pair(partitionStreamPtr->GetCluster(), partitionStreamPtr->GetPartitionId()); + } + + void SubscribeOnNextEvent() { if (!SubscribedOnEvent) { SubscribedOnEvent = true; NActors::TActorSystem* actorSystem = NActors::TActivationContext::ActorSystem(); @@ -300,129 +300,129 @@ private: actorSystem->Send(selfId, new TEvPrivate::TEvSourceDataReady()); }); } - } - - struct TPQEventProcessor { - void operator()(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& event) { - for (const auto& message : event.GetMessages()) { - const TString& data = message.GetData(); - - LWPROBE(PqReadDataReceived, Self.TxId, Self.SourceParams.GetTopicPath(), data); - SINK_LOG_T("Data received: " << data); - + } + + struct TPQEventProcessor { + void operator()(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& event) { + for (const auto& message : event.GetMessages()) { + const TString& data = message.GetData(); + + LWPROBE(PqReadDataReceived, Self.TxId, Self.SourceParams.GetTopicPath(), data); + SINK_LOG_T("Data received: " << data); + Batch.emplace_back(NKikimr::NMiniKQL::MakeString(NUdf::TStringRef(data.Data(), data.Size()))); UsedSpace += data.Size(); - } - Self.UpdateStateWithNewReadData(event); - Self.CurrentDeferredCommit.Add(event); - } - - void operator()(NYdb::NPersQueue::TSessionClosedEvent& ev) { - ythrow yexception() << "Read session to topic \"" << Self.SourceParams.GetTopicPath() - << "\" was closed: " << ev.DebugString(); - } - - void operator()(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&) { } - - void operator()(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent& event) { - TMaybe<ui64> readOffset; - const auto offsetIt = Self.PartitionToOffset.find(MakePartitionKey(event.GetPartitionStream())); - if (offsetIt != Self.PartitionToOffset.end()) { - readOffset = offsetIt->second; - } - event.Confirm(readOffset); - } - - void operator()(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent& event) { - event.Confirm(); - } - - void operator()(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&) { } - - void operator()(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&) { } - - TDqPqReadActor& Self; - TUnboxedValueVector& Batch; - i64& UsedSpace; - }; - -private: - const ui64 InputIndex; - const TString TxId; + } + Self.UpdateStateWithNewReadData(event); + Self.CurrentDeferredCommit.Add(event); + } + + void operator()(NYdb::NPersQueue::TSessionClosedEvent& ev) { + ythrow yexception() << "Read session to topic \"" << Self.SourceParams.GetTopicPath() + << "\" was closed: " << ev.DebugString(); + } + + void operator()(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&) { } + + void operator()(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent& event) { + TMaybe<ui64> readOffset; + const auto offsetIt = Self.PartitionToOffset.find(MakePartitionKey(event.GetPartitionStream())); + if (offsetIt != Self.PartitionToOffset.end()) { + readOffset = offsetIt->second; + } + event.Confirm(readOffset); + } + + void operator()(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent& event) { + event.Confirm(); + } + + void operator()(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&) { } + + void operator()(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&) { } + + TDqPqReadActor& Self; + TUnboxedValueVector& Batch; + i64& UsedSpace; + }; + +private: + const ui64 InputIndex; + const TString TxId; const i64 BufferSize; - const THolderFactory& HolderFactory; + const THolderFactory& HolderFactory; NYdb::TDriver Driver; std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactory; - const NPq::NProto::TDqPqTopicSource SourceParams; - const NPq::NProto::TDqReadTaskParams ReadParams; - std::unique_ptr<NYdb::NPersQueue::TPersQueueClient> PersQueueClient; - std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession; - NThreading::TFuture<void> EventFuture; - THashMap<TPartitionKey, ui64> PartitionToOffset; // {cluster, partition} -> offset of next event. + const NPq::NProto::TDqPqTopicSource SourceParams; + const NPq::NProto::TDqReadTaskParams ReadParams; + std::unique_ptr<NYdb::NPersQueue::TPersQueueClient> PersQueueClient; + std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession; + NThreading::TFuture<void> EventFuture; + THashMap<TPartitionKey, ui64> PartitionToOffset; // {cluster, partition} -> offset of next event. TInstant StartingMessageTimestamp; ICallbacks* const Callbacks; - std::queue<std::pair<ui64, NYdb::NPersQueue::TDeferredCommit>> DeferredCommits; - NYdb::NPersQueue::TDeferredCommit CurrentDeferredCommit; + std::queue<std::pair<ui64, NYdb::NPersQueue::TDeferredCommit>> DeferredCommits; + NYdb::NPersQueue::TDeferredCommit CurrentDeferredCommit; bool SubscribedOnEvent = false; -}; - +}; + std::pair<IDqSourceActor*, NActors::IActor*> CreateDqPqReadActor( NPq::NProto::TDqPqTopicSource&& settings, - ui64 inputIndex, - TTxId txId, - const THashMap<TString, TString>& secureParams, - const THashMap<TString, TString>& taskParams, + ui64 inputIndex, + TTxId txId, + const THashMap<TString, TString>& secureParams, + const THashMap<TString, TString>& taskParams, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IDqSourceActor::ICallbacks* callbacks, - const NKikimr::NMiniKQL::THolderFactory& holderFactory, + const NKikimr::NMiniKQL::THolderFactory& holderFactory, i64 bufferSize ) -{ - auto taskParamsIt = taskParams.find("pq"); - YQL_ENSURE(taskParamsIt != taskParams.end(), "Failed to get pq task params"); - +{ + auto taskParamsIt = taskParams.find("pq"); + YQL_ENSURE(taskParamsIt != taskParams.end(), "Failed to get pq task params"); + NPq::NProto::TDqReadTaskParams readTaskParamsMsg; - YQL_ENSURE(readTaskParamsMsg.ParseFromString(taskParamsIt->second), "Failed to parse DqPqRead task params"); - + YQL_ENSURE(readTaskParamsMsg.ParseFromString(taskParamsIt->second), "Failed to parse DqPqRead task params"); + const TString& tokenName = settings.GetToken().GetName(); const TString token = secureParams.Value(tokenName, TString()); - const bool addBearerToToken = settings.GetAddBearerToToken(); + const bool addBearerToToken = settings.GetAddBearerToToken(); - TDqPqReadActor* actor = new TDqPqReadActor( - inputIndex, - std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId), + TDqPqReadActor* actor = new TDqPqReadActor( + inputIndex, + std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId), holderFactory, - std::move(settings), - std::move(readTaskParamsMsg), + std::move(settings), + std::move(readTaskParamsMsg), std::move(driver), - CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken), + CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken), callbacks, bufferSize ); - return {actor, actor}; -} - + return {actor, actor}; +} + void RegisterDqPqReadActorFactory(TDqSourceFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { factory.Register<NPq::NProto::TDqPqTopicSource>("PqSource", - [driver = std::move(driver), credentialsFactory = std::move(credentialsFactory)]( - NPq::NProto::TDqPqTopicSource&& settings, - IDqSourceActorFactory::TArguments&& args) - { - NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(DQ_PQ_PROVIDER)); - return CreateDqPqReadActor( - std::move(settings), - args.InputIndex, - args.TxId, - args.SecureParams, - args.TaskParams, - driver, - credentialsFactory, - args.Callback, - args.HolderFactory); + [driver = std::move(driver), credentialsFactory = std::move(credentialsFactory)]( + NPq::NProto::TDqPqTopicSource&& settings, + IDqSourceActorFactory::TArguments&& args) + { + NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(DQ_PQ_PROVIDER)); + return CreateDqPqReadActor( + std::move(settings), + args.InputIndex, + args.TxId, + args.SecureParams, + args.TaskParams, + driver, + credentialsFactory, + args.Callback, + args.HolderFactory); }); } -} // namespace NYql::NDq +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h index 06999d9efe2..d7e2c82dd04 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> @@ -8,33 +8,33 @@ #include <ydb/library/yql/providers/pq/proto/dq_io.pb.h> #include <ydb/library/yql/providers/pq/proto/dq_task_params.pb.h> - + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - -#include <library/cpp/actors/core/actor.h> - -#include <util/generic/size_literals.h> -#include <util/system/types.h> - - -namespace NYql::NDq { + +#include <library/cpp/actors/core/actor.h> + +#include <util/generic/size_literals.h> +#include <util/system/types.h> + + +namespace NYql::NDq { class TDqSourceFactory; - -const i64 PQReadDefaultFreeSpace = 16_MB; - + +const i64 PQReadDefaultFreeSpace = 16_MB; + std::pair<IDqSourceActor*, NActors::IActor*> CreateDqPqReadActor( NPq::NProto::TDqPqTopicSource&& settings, - ui64 inputIndex, - TTxId txId, - const THashMap<TString, TString>& secureParams, - const THashMap<TString, TString>& taskParams, + ui64 inputIndex, + TTxId txId, + const THashMap<TString, TString>& secureParams, + const THashMap<TString, TString>& taskParams, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IDqSourceActor::ICallbacks* callback, - const NKikimr::NMiniKQL::THolderFactory& holderFactory, + const NKikimr::NMiniKQL::THolderFactory& holderFactory, i64 bufferSize = PQReadDefaultFreeSpace ); - + void RegisterDqPqReadActorFactory(TDqSourceFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); -} // namespace NYql::NDq +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp index 5c8242229a4..85fbc91c01d 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp @@ -1,177 +1,177 @@ -#include "dq_pq_write_actor.h" -#include "probes.h" - +#include "dq_pq_write_actor.h" +#include "probes.h" + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> #include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> #include <ydb/library/yql/dq/common/dq_common.h> #include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h> - + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h> #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/providers/pq/proto/dq_io_state.pb.h> #include <ydb/library/yql/utils/yql_panic.h> - + #include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h> #include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h> -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> -#include <library/cpp/lwtrace/mon/mon_lwtrace.h> - -#include <util/generic/algorithm.h> -#include <util/generic/hash.h> +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> +#include <library/cpp/lwtrace/mon/mon_lwtrace.h> + +#include <util/generic/algorithm.h> +#include <util/generic/hash.h> #include <util/string/builder.h> - -#include <algorithm> -#include <queue> -#include <variant> - -namespace NKikimrServices { + +#include <algorithm> +#include <queue> +#include <variant> + +namespace NKikimrServices { // using constant value from ydb/core/protos/services.proto // but to avoid peerdir on ydb/core/protos we introduce this constant - constexpr ui32 KQP_COMPUTE = 535; -}; - -const TString LogPrefix = "PQ sink. "; - -#define SINK_LOG_T(s) \ - LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_D(s) \ - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_I(s) \ - LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_W(s) \ - LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_N(s) \ - LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_E(s) \ - LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG_C(s) \ - LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) -#define SINK_LOG(prio, s) \ - LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, LogPrefix << s) - -namespace NYql::NDq { - -using namespace NActors; -using namespace NLog; -using namespace NKikimr::NMiniKQL; - -constexpr ui32 StateVersion = 1; -constexpr ui32 MaxMessageSize = 1_MB; - -namespace { - -LWTRACE_USING(DQ_PQ_PROVIDER); - -struct TEvPrivate { - // Event ids - enum EEv : ui32 { - EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), - - EvPqEventsReady = EvBegin, - - EvEnd - }; - - static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)"); - - // Events - - struct TEvPqEventsReady : public TEventLocal<TEvPqEventsReady, EvPqEventsReady> {}; -}; - -} // namespace - -class TDqPqWriteActor : public NActors::TActor<TDqPqWriteActor>, public IDqSinkActor { -public: - TDqPqWriteActor( - ui64 outputIndex, - const TString& txId, - NPq::NProto::TDqPqTopicSink&& sinkParams, + constexpr ui32 KQP_COMPUTE = 535; +}; + +const TString LogPrefix = "PQ sink. "; + +#define SINK_LOG_T(s) \ + LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_D(s) \ + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_I(s) \ + LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_W(s) \ + LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_N(s) \ + LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_E(s) \ + LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG_C(s) \ + LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s) +#define SINK_LOG(prio, s) \ + LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, LogPrefix << s) + +namespace NYql::NDq { + +using namespace NActors; +using namespace NLog; +using namespace NKikimr::NMiniKQL; + +constexpr ui32 StateVersion = 1; +constexpr ui32 MaxMessageSize = 1_MB; + +namespace { + +LWTRACE_USING(DQ_PQ_PROVIDER); + +struct TEvPrivate { + // Event ids + enum EEv : ui32 { + EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), + + EvPqEventsReady = EvBegin, + + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)"); + + // Events + + struct TEvPqEventsReady : public TEventLocal<TEvPqEventsReady, EvPqEventsReady> {}; +}; + +} // namespace + +class TDqPqWriteActor : public NActors::TActor<TDqPqWriteActor>, public IDqSinkActor { +public: + TDqPqWriteActor( + ui64 outputIndex, + const TString& txId, + NPq::NProto::TDqPqTopicSink&& sinkParams, NYdb::TDriver driver, std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, IDqSinkActor::ICallbacks* callbacks, - i64 freeSpace) - : TActor<TDqPqWriteActor>(&TDqPqWriteActor::StateFunc) - , OutputIndex(outputIndex) - , TxId(txId) - , SinkParams(std::move(sinkParams)) + i64 freeSpace) + : TActor<TDqPqWriteActor>(&TDqPqWriteActor::StateFunc) + , OutputIndex(outputIndex) + , TxId(txId) + , SinkParams(std::move(sinkParams)) , Driver(std::move(driver)) , CredentialsProviderFactory(credentialsProviderFactory) - , Callbacks(callbacks) - , FreeSpace(freeSpace) + , Callbacks(callbacks) + , FreeSpace(freeSpace) , PersQueueClient(Driver, GetPersQueueClientSettings()) - { } - + { } + static constexpr char ActorName[] = "DQ_PQ_WRITE_ACTOR"; -public: +public: void SendData( - NKikimr::NMiniKQL::TUnboxedValueVector&& batch, + NKikimr::NMiniKQL::TUnboxedValueVector&& batch, i64 dataSize, const TMaybe<NDqProto::TCheckpoint>& checkpoint, bool finished) override - { + { Y_UNUSED(finished); - Y_UNUSED(dataSize); - - CreateSessionIfNotExists(); - + Y_UNUSED(dataSize); + + CreateSessionIfNotExists(); + for (const NUdf::TUnboxedValue& item : batch) { - if (!item.IsBoxed()) { - Fail("Struct with single field was expected"); - return; - } - + if (!item.IsBoxed()) { + Fail("Struct with single field was expected"); + return; + } + const NUdf::TUnboxedValue dataCol = item.GetElement(0); - - if (!dataCol.IsString() && !dataCol.IsEmbedded()) { - Fail(TStringBuilder() << "Non string value could not be written to YDS stream"); - return; - } - + + if (!dataCol.IsString() && !dataCol.IsEmbedded()) { + Fail(TStringBuilder() << "Non string value could not be written to YDS stream"); + return; + } + TString data(dataCol.AsStringRef()); - - LWPROBE(PqWriteDataToSend, TxId, SinkParams.GetTopicPath(), data); - SINK_LOG_T("Received data for sending: " << data); - - const auto messageSize = GetItemSize(data); - if (messageSize > MaxMessageSize) { - Fail(TStringBuilder() << "Max message size for YDS is " << MaxMessageSize - << " bytes but received message with size of " << messageSize << " bytes"); - return; - } - - FreeSpace -= messageSize; - Buffer.push(std::move(data)); - } - - if (checkpoint) { - if (Buffer.empty()) { + + LWPROBE(PqWriteDataToSend, TxId, SinkParams.GetTopicPath(), data); + SINK_LOG_T("Received data for sending: " << data); + + const auto messageSize = GetItemSize(data); + if (messageSize > MaxMessageSize) { + Fail(TStringBuilder() << "Max message size for YDS is " << MaxMessageSize + << " bytes but received message with size of " << messageSize << " bytes"); + return; + } + + FreeSpace -= messageSize; + Buffer.push(std::move(data)); + } + + if (checkpoint) { + if (Buffer.empty()) { Callbacks->OnSinkStateSaved(BuildState(), OutputIndex, *checkpoint); - } else { - DeferredCheckpoints.emplace(NextSeqNo + Buffer.size() - 1, *checkpoint); - } - } - - if (!Buffer.empty() && ContinuationToken) { - WriteNextMessage(std::move(*ContinuationToken)); - ContinuationToken = std::nullopt; - } - - while (HandleNewPQEvents()) { } // Write messages while new continuationTokens are arriving - - if (FreeSpace <= 0) { - ShouldNotifyNewFreeSpace = true; - } - }; - + } else { + DeferredCheckpoints.emplace(NextSeqNo + Buffer.size() - 1, *checkpoint); + } + } + + if (!Buffer.empty() && ContinuationToken) { + WriteNextMessage(std::move(*ContinuationToken)); + ContinuationToken = std::nullopt; + } + + while (HandleNewPQEvents()) { } // Write messages while new continuationTokens are arriving + + if (FreeSpace <= 0) { + ShouldNotifyNewFreeSpace = true; + } + }; + void LoadState(const NDqProto::TSinkState& state) override { Y_VERIFY(NextSeqNo == 1); const auto& data = state.GetData().GetStateData(); @@ -180,112 +180,112 @@ public: YQL_ENSURE(stateProto.ParseFromString(data.GetBlob()), "Serialized state is corrupted"); SourceId = stateProto.GetSourceId(); ConfirmedSeqNo = stateProto.GetConfirmedSeqNo(); - NextSeqNo = ConfirmedSeqNo + 1; + NextSeqNo = ConfirmedSeqNo + 1; return; - } + } ythrow yexception() << "Invalid state version " << data.GetVersion(); - } - - void CommitState(const NDqProto::TCheckpoint& checkpoint) override { - Y_UNUSED(checkpoint); - } - - i64 GetFreeSpace() const override { - return FreeSpace; - }; - - ui64 GetOutputIndex() const override { - return OutputIndex; - }; - -private: - STRICT_STFUNC(StateFunc, - hFunc(TEvPrivate::TEvPqEventsReady, Handle); - ) - - void Handle(TEvPrivate::TEvPqEventsReady::TPtr&) { - while (HandleNewPQEvents()) { } - SubscribeOnNextEvent(); - } - + } + + void CommitState(const NDqProto::TCheckpoint& checkpoint) override { + Y_UNUSED(checkpoint); + } + + i64 GetFreeSpace() const override { + return FreeSpace; + }; + + ui64 GetOutputIndex() const override { + return OutputIndex; + }; + +private: + STRICT_STFUNC(StateFunc, + hFunc(TEvPrivate::TEvPqEventsReady, Handle); + ) + + void Handle(TEvPrivate::TEvPqEventsReady::TPtr&) { + while (HandleNewPQEvents()) { } + SubscribeOnNextEvent(); + } + // IActor & IDqSinkActor void PassAway() override { // Is called from Compute Actor - if (WriteSession) { - WriteSession->Close(TDuration::Zero()); - } + if (WriteSession) { + WriteSession->Close(TDuration::Zero()); + } TActor<TDqPqWriteActor>::PassAway(); - } - -private: - const TString& GetSourceId() { - if (!SourceId) { - SourceId = CreateGuidAsString(); // Not loaded from state, so this is the first run. - } - return SourceId; - } - - NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() { - return NYdb::NPersQueue::TWriteSessionSettings(SinkParams.GetTopicPath(), GetSourceId()) - .MaxMemoryUsage(FreeSpace) - .ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Auto) - .Codec(SinkParams.GetClusterType() == NPq::NProto::DataStreams - ? NYdb::NPersQueue::ECodec::RAW - : NYdb::NPersQueue::ECodec::GZIP); - } - + } + +private: + const TString& GetSourceId() { + if (!SourceId) { + SourceId = CreateGuidAsString(); // Not loaded from state, so this is the first run. + } + return SourceId; + } + + NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() { + return NYdb::NPersQueue::TWriteSessionSettings(SinkParams.GetTopicPath(), GetSourceId()) + .MaxMemoryUsage(FreeSpace) + .ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Auto) + .Codec(SinkParams.GetClusterType() == NPq::NProto::DataStreams + ? NYdb::NPersQueue::ECodec::RAW + : NYdb::NPersQueue::ECodec::GZIP); + } + NYdb::NPersQueue::TPersQueueClientSettings GetPersQueueClientSettings() { return NYdb::NPersQueue::TPersQueueClientSettings() - .Database(SinkParams.GetDatabase()) - .DiscoveryEndpoint(SinkParams.GetEndpoint()) - .EnableSsl(SinkParams.GetUseSsl()) + .Database(SinkParams.GetDatabase()) + .DiscoveryEndpoint(SinkParams.GetEndpoint()) + .EnableSsl(SinkParams.GetUseSsl()) .CredentialsProviderFactory(CredentialsProviderFactory); - } - - static i64 GetItemSize(const TString& item) { - return std::max(static_cast<i64>(item.size()), static_cast<i64>(1)); - } - - void CreateSessionIfNotExists() { - if (!WriteSession) { - WriteSession = PersQueueClient.CreateWriteSession(GetWriteSessionSettings()); - SubscribeOnNextEvent(); - } - } - - void SubscribeOnNextEvent() { - if (!WriteSession) { - return; - } - - NActors::TActorSystem* actorSystem = NActors::TActivationContext::ActorSystem(); - EventFuture = WriteSession->WaitEvent().Subscribe([actorSystem, selfId = SelfId()](const auto&){ - actorSystem->Send(selfId, new TEvPrivate::TEvPqEventsReady()); - }); - } - - bool HandleNewPQEvents() { - if (!WriteSession) { - return false; - } - - auto events = WriteSession->GetEvents(); - for (auto& event : events) { + } + + static i64 GetItemSize(const TString& item) { + return std::max(static_cast<i64>(item.size()), static_cast<i64>(1)); + } + + void CreateSessionIfNotExists() { + if (!WriteSession) { + WriteSession = PersQueueClient.CreateWriteSession(GetWriteSessionSettings()); + SubscribeOnNextEvent(); + } + } + + void SubscribeOnNextEvent() { + if (!WriteSession) { + return; + } + + NActors::TActorSystem* actorSystem = NActors::TActivationContext::ActorSystem(); + EventFuture = WriteSession->WaitEvent().Subscribe([actorSystem, selfId = SelfId()](const auto&){ + actorSystem->Send(selfId, new TEvPrivate::TEvPqEventsReady()); + }); + } + + bool HandleNewPQEvents() { + if (!WriteSession) { + return false; + } + + auto events = WriteSession->GetEvents(); + for (auto& event : events) { auto issues = std::visit(TPQEventProcessor{*this}, event); if (issues) { - WriteSession->Close(TDuration::Zero()); - WriteSession.reset(); - Callbacks->OnSinkError(OutputIndex, *issues, true); - break; - } - - if (ShouldNotifyNewFreeSpace && FreeSpace > 0) { - Callbacks->ResumeExecution(); - ShouldNotifyNewFreeSpace = false; - } - } - return !events.empty(); - } - + WriteSession->Close(TDuration::Zero()); + WriteSession.reset(); + Callbacks->OnSinkError(OutputIndex, *issues, true); + break; + } + + if (ShouldNotifyNewFreeSpace && FreeSpace > 0) { + Callbacks->ResumeExecution(); + ShouldNotifyNewFreeSpace = false; + } + } + return !events.empty(); + } + NDqProto::TSinkState BuildState() { NPq::NProto::TDqPqTopicSinkState stateProto; stateProto.SetSourceId(GetSourceId()); @@ -298,138 +298,138 @@ private: data->SetVersion(StateVersion); data->SetBlob(serializedState); return sinkState; - } - - void WriteNextMessage(NYdb::NPersQueue::TContinuationToken&& token) { - WriteSession->Write(std::move(token), Buffer.front(), NextSeqNo++); - WaitingAcks.push(GetItemSize(Buffer.front())); - Buffer.pop(); - } - - void Fail(TString message) { - TIssues issues; - issues.AddIssue(message); - Callbacks->OnSinkError(OutputIndex, issues, true); - return; - } - - struct TPQEventProcessor { + } + + void WriteNextMessage(NYdb::NPersQueue::TContinuationToken&& token) { + WriteSession->Write(std::move(token), Buffer.front(), NextSeqNo++); + WaitingAcks.push(GetItemSize(Buffer.front())); + Buffer.pop(); + } + + void Fail(TString message) { + TIssues issues; + issues.AddIssue(message); + Callbacks->OnSinkError(OutputIndex, issues, true); + return; + } + + struct TPQEventProcessor { std::optional<TIssues> operator()(NYdb::NPersQueue::TSessionClosedEvent& ev) { TIssues issues; - issues.AddIssue(TStringBuilder() << "Write session to topic \"" << Self.SinkParams.GetTopicPath() << "\" was closed: " << ev.DebugString()); + issues.AddIssue(TStringBuilder() << "Write session to topic \"" << Self.SinkParams.GetTopicPath() << "\" was closed: " << ev.DebugString()); return issues; - } - + } + std::optional<TIssues> operator()(NYdb::NPersQueue::TWriteSessionEvent::TAcksEvent& ev) { - if (ev.Acks.empty()) { - return std::nullopt; - } - - //Y_VERIFY(Self.ConfirmedSeqNo == 0 || ev.Acks.front().SeqNo == Self.ConfirmedSeqNo + 1); - - for (auto it = ev.Acks.begin(); it != ev.Acks.end(); ++it) { - //Y_VERIFY(it == ev.Acks.begin() || it->SeqNo == std::prev(it)->SeqNo + 1); - - if (it->State == NYdb::NPersQueue::TWriteSessionEvent::TWriteAck::EEventState::EES_DISCARDED) { + if (ev.Acks.empty()) { + return std::nullopt; + } + + //Y_VERIFY(Self.ConfirmedSeqNo == 0 || ev.Acks.front().SeqNo == Self.ConfirmedSeqNo + 1); + + for (auto it = ev.Acks.begin(); it != ev.Acks.end(); ++it) { + //Y_VERIFY(it == ev.Acks.begin() || it->SeqNo == std::prev(it)->SeqNo + 1); + + if (it->State == NYdb::NPersQueue::TWriteSessionEvent::TWriteAck::EEventState::EES_DISCARDED) { TIssues issues; issues.AddIssue(TStringBuilder() << "Message with seqNo " << it->SeqNo << " was discarded"); return issues; - } - - Self.FreeSpace += Self.WaitingAcks.front(); - Self.WaitingAcks.pop(); - - if (!Self.DeferredCheckpoints.empty() && std::get<0>(Self.DeferredCheckpoints.front()) == it->SeqNo) { - Self.ConfirmedSeqNo = it->SeqNo; + } + + Self.FreeSpace += Self.WaitingAcks.front(); + Self.WaitingAcks.pop(); + + if (!Self.DeferredCheckpoints.empty() && std::get<0>(Self.DeferredCheckpoints.front()) == it->SeqNo) { + Self.ConfirmedSeqNo = it->SeqNo; Self.Callbacks->OnSinkStateSaved(Self.BuildState(), Self.OutputIndex, std::get<1>(Self.DeferredCheckpoints.front())); - Self.DeferredCheckpoints.pop(); - } - } - Self.ConfirmedSeqNo = ev.Acks.back().SeqNo; - - return std::nullopt; - } - + Self.DeferredCheckpoints.pop(); + } + } + Self.ConfirmedSeqNo = ev.Acks.back().SeqNo; + + return std::nullopt; + } + std::optional<TIssues> operator()(NYdb::NPersQueue::TWriteSessionEvent::TReadyToAcceptEvent& ev) { //Y_VERIFY(!Self.ContinuationToken); - - if (!Self.Buffer.empty()) { - Self.WriteNextMessage(std::move(ev.ContinuationToken)); - return std::nullopt; - } - - Self.ContinuationToken = std::move(ev.ContinuationToken); - return std::nullopt; - } - - TDqPqWriteActor& Self; - }; - -private: - const ui64 OutputIndex; - const TString TxId; - const NPq::NProto::TDqPqTopicSink SinkParams; + + if (!Self.Buffer.empty()) { + Self.WriteNextMessage(std::move(ev.ContinuationToken)); + return std::nullopt; + } + + Self.ContinuationToken = std::move(ev.ContinuationToken); + return std::nullopt; + } + + TDqPqWriteActor& Self; + }; + +private: + const ui64 OutputIndex; + const TString TxId; + const NPq::NProto::TDqPqTopicSink SinkParams; NYdb::TDriver Driver; std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactory; IDqSinkActor::ICallbacks* const Callbacks; - i64 FreeSpace = 0; - - NYdb::NPersQueue::TPersQueueClient PersQueueClient; - std::shared_ptr<NYdb::NPersQueue::IWriteSession> WriteSession; - TString SourceId; - ui64 NextSeqNo = 1; - ui64 ConfirmedSeqNo = 0; - std::optional<NYdb::NPersQueue::TContinuationToken> ContinuationToken; - NThreading::TFuture<void> EventFuture; - std::queue<i64> InflightMessageSizes; - bool ShouldNotifyNewFreeSpace = false; - std::queue<TString> Buffer; - std::queue<i64> WaitingAcks; // Size of items which are waiting for acks (used to update free space) - std::queue<std::tuple<ui64, NDqProto::TCheckpoint>> DeferredCheckpoints; -}; - + i64 FreeSpace = 0; + + NYdb::NPersQueue::TPersQueueClient PersQueueClient; + std::shared_ptr<NYdb::NPersQueue::IWriteSession> WriteSession; + TString SourceId; + ui64 NextSeqNo = 1; + ui64 ConfirmedSeqNo = 0; + std::optional<NYdb::NPersQueue::TContinuationToken> ContinuationToken; + NThreading::TFuture<void> EventFuture; + std::queue<i64> InflightMessageSizes; + bool ShouldNotifyNewFreeSpace = false; + std::queue<TString> Buffer; + std::queue<i64> WaitingAcks; // Size of items which are waiting for acks (used to update free space) + std::queue<std::tuple<ui64, NDqProto::TCheckpoint>> DeferredCheckpoints; +}; + std::pair<IDqSinkActor*, NActors::IActor*> CreateDqPqWriteActor( NPq::NProto::TDqPqTopicSink&& settings, - ui64 outputIndex, - TTxId txId, - const THashMap<TString, TString>& secureParams, + ui64 outputIndex, + TTxId txId, + const THashMap<TString, TString>& secureParams, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IDqSinkActor::ICallbacks* callbacks, - i64 freeSpace) -{ + i64 freeSpace) +{ const TString& tokenName = settings.GetToken().GetName(); const TString token = secureParams.Value(tokenName, TString()); - const bool addBearerToToken = settings.GetAddBearerToToken(); + const bool addBearerToToken = settings.GetAddBearerToToken(); - TDqPqWriteActor* actor = new TDqPqWriteActor( - outputIndex, - std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId), - std::move(settings), + TDqPqWriteActor* actor = new TDqPqWriteActor( + outputIndex, + std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId), + std::move(settings), std::move(driver), - CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken), - callbacks, - freeSpace); - return {actor, actor}; -} - + CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken), + callbacks, + freeSpace); + return {actor, actor}; +} + void RegisterDqPqWriteActorFactory(TDqSinkFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { factory.Register<NPq::NProto::TDqPqTopicSink>("PqSink", [driver = std::move(driver), credentialsFactory = std::move(credentialsFactory)]( - NPq::NProto::TDqPqTopicSink&& settings, - IDqSinkActorFactory::TArguments&& args) - { - NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(DQ_PQ_PROVIDER)); + NPq::NProto::TDqPqTopicSink&& settings, + IDqSinkActorFactory::TArguments&& args) + { + NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(DQ_PQ_PROVIDER)); return CreateDqPqWriteActor( std::move(settings), - args.OutputIndex, - args.TxId, - args.SecureParams, + args.OutputIndex, + args.TxId, + args.SecureParams, driver, credentialsFactory, - args.Callback + args.Callback ); - }); + }); } -} // namespace NYql::NDq +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h index 90b393dfcc9..6736c8188b2 100644 --- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h +++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> @@ -8,28 +8,28 @@ #include <ydb/library/yql/providers/pq/proto/dq_io.pb.h> #include <ydb/library/yql/providers/pq/proto/dq_task_params.pb.h> - + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - -#include <library/cpp/actors/core/actor.h> - -#include <util/generic/size_literals.h> -#include <util/system/types.h> - -namespace NYql::NDq { - -constexpr i64 DqPqDefaultFreeSpace = 16_MB; - + +#include <library/cpp/actors/core/actor.h> + +#include <util/generic/size_literals.h> +#include <util/system/types.h> + +namespace NYql::NDq { + +constexpr i64 DqPqDefaultFreeSpace = 16_MB; + std::pair<IDqSinkActor*, NActors::IActor*> CreateDqPqWriteActor( NPq::NProto::TDqPqTopicSink&& settings, - ui64 outputIndex, - TTxId txId, - const THashMap<TString, TString>& secureParams, + ui64 outputIndex, + TTxId txId, + const THashMap<TString, TString>& secureParams, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IDqSinkActor::ICallbacks* callbacks, - i64 freeSpace = DqPqDefaultFreeSpace); - + i64 freeSpace = DqPqDefaultFreeSpace); + void RegisterDqPqWriteActorFactory(TDqSinkFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); -} // namespace NYql::NDq +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/pq/async_io/probes.cpp b/ydb/library/yql/providers/pq/async_io/probes.cpp index 5cafed75a5e..904821f4186 100644 --- a/ydb/library/yql/providers/pq/async_io/probes.cpp +++ b/ydb/library/yql/providers/pq/async_io/probes.cpp @@ -1,3 +1,3 @@ -#include "probes.h" - -LWTRACE_DEFINE_PROVIDER(DQ_PQ_PROVIDER) +#include "probes.h" + +LWTRACE_DEFINE_PROVIDER(DQ_PQ_PROVIDER) diff --git a/ydb/library/yql/providers/pq/async_io/probes.h b/ydb/library/yql/providers/pq/async_io/probes.h index 1863894ca73..7bd876dc4d3 100644 --- a/ydb/library/yql/providers/pq/async_io/probes.h +++ b/ydb/library/yql/providers/pq/async_io/probes.h @@ -1,17 +1,17 @@ -#pragma once - -#include <library/cpp/lwtrace/all.h> - -#define DQ_PQ_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \ - PROBE(PqReadDataReceived, \ - GROUPS(), \ - TYPES(TString, TString, TString), \ - NAMES("queryId", "topic", "data")) \ - PROBE(PqWriteDataToSend, \ - GROUPS(), \ - TYPES(TString, TString, TString), \ - NAMES("queryId", "topic", "data")) \ - -// DQ_PQ_PROVIDER - -LWTRACE_DECLARE_PROVIDER(DQ_PQ_PROVIDER) +#pragma once + +#include <library/cpp/lwtrace/all.h> + +#define DQ_PQ_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \ + PROBE(PqReadDataReceived, \ + GROUPS(), \ + TYPES(TString, TString, TString), \ + NAMES("queryId", "topic", "data")) \ + PROBE(PqWriteDataToSend, \ + GROUPS(), \ + TYPES(TString, TString, TString), \ + NAMES("queryId", "topic", "data")) \ + +// DQ_PQ_PROVIDER + +LWTRACE_DECLARE_PROVIDER(DQ_PQ_PROVIDER) diff --git a/ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp b/ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp index 7ddef8bda4a..ed6579022cf 100644 --- a/ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp +++ b/ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp @@ -1,169 +1,169 @@ -#include "ut_helpers.h" - +#include "ut_helpers.h" + #include <ydb/library/yql/utils/yql_panic.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <thread> - -namespace NYql::NDq { - -Y_UNIT_TEST_SUITE(TDqPqReadActorTest) { + +#include <library/cpp/testing/unittest/registar.h> + +#include <thread> + +namespace NYql::NDq { + +Y_UNIT_TEST_SUITE(TDqPqReadActorTest) { Y_UNIT_TEST_F(TestReadFromTopic, TPqIoTestFixture) { - const TString topicName = "ReadFromTopic"; + const TString topicName = "ReadFromTopic"; InitSource(topicName); - - const std::vector<TString> data = { "1", "2", "3", "4" }; - PQWrite(data, topicName); - + + const std::vector<TString> data = { "1", "2", "3", "4" }; + PQWrite(data, topicName); + auto result = SourceReadUntil<TString>(UVParser, 4); - UNIT_ASSERT_EQUAL(result, data); - } - + UNIT_ASSERT_EQUAL(result, data); + } + Y_UNIT_TEST_F(ReadWithFreeSpace, TPqIoTestFixture) { - const TString topicName = "ReadWithFreeSpace"; + const TString topicName = "ReadWithFreeSpace"; InitSource(topicName); - - PQWrite({"data1", "data2", "data3"}, topicName); - - { + + PQWrite({"data1", "data2", "data3"}, topicName); + + { auto result = SourceReadUntil<TString>(UVParser, 1, 1); - std::vector<TString> expected {"data1"}; - UNIT_ASSERT_EQUAL(result, expected); - } - + std::vector<TString> expected {"data1"}; + UNIT_ASSERT_EQUAL(result, expected); + } + UNIT_ASSERT_EQUAL(SourceRead<TString>(UVParser, 0).size(), 0); UNIT_ASSERT_EQUAL(SourceRead<TString>(UVParser, -1).size(), 0); - } - + } + Y_UNIT_TEST_F(ReadNonExistentTopic, TPqIoTestFixture) { - const TString topicName = "NonExistentTopic"; + const TString topicName = "NonExistentTopic"; InitSource(topicName); - - while (true) { - try { + + while (true) { + try { SourceRead<TString>(UVParser); - } catch (yexception& e) { - UNIT_ASSERT_STRING_CONTAINS(e.what(), "Read session to topic \"NonExistentTopic\" was closed"); - break; - } - - sleep(1); - } - } - - Y_UNIT_TEST(TestSaveLoadPqRead) { + } catch (yexception& e) { + UNIT_ASSERT_STRING_CONTAINS(e.what(), "Read session to topic \"NonExistentTopic\" was closed"); + break; + } + + sleep(1); + } + } + + Y_UNIT_TEST(TestSaveLoadPqRead) { NDqProto::TSourceState state; - const TString topicName = "SaveLoadPqRead"; - - { + const TString topicName = "SaveLoadPqRead"; + + { TPqIoTestFixture setup1; setup1.InitSource(topicName); - - std::vector<TString> data {"data"}; - PQWrite(data, topicName); - - auto result = setup1.SourceReadUntil<TString>(UVParser, 1); - UNIT_ASSERT_EQUAL(result, data); - - auto checkpoint = CreateCheckpoint(); + + std::vector<TString> data {"data"}; + PQWrite(data, topicName); + + auto result = setup1.SourceReadUntil<TString>(UVParser, 1); + UNIT_ASSERT_EQUAL(result, data); + + auto checkpoint = CreateCheckpoint(); setup1.SaveSourceState(checkpoint, state); - Cerr << "State saved" << Endl; - } - + Cerr << "State saved" << Endl; + } + NDqProto::TSourceState state2; - { + { TPqIoTestFixture setup2; setup2.InitSource(topicName); - - std::vector<TString> data {"data"}; - PQWrite({"data"}, topicName); - - setup2.LoadSource(state); - Cerr << "State loaded" << Endl; - auto result = setup2.SourceReadUntil<TString>(UVParser, 1); - UNIT_ASSERT_EQUAL(result, data); - - auto checkpoint = CreateCheckpoint(); + + std::vector<TString> data {"data"}; + PQWrite({"data"}, topicName); + + setup2.LoadSource(state); + Cerr << "State loaded" << Endl; + auto result = setup2.SourceReadUntil<TString>(UVParser, 1); + UNIT_ASSERT_EQUAL(result, data); + + auto checkpoint = CreateCheckpoint(); setup2.SaveSourceState(checkpoint, state2); - - PQWrite({"futherData"}, topicName); - } - + + PQWrite({"futherData"}, topicName); + } + NDqProto::TSourceState state3; - { + { TPqIoTestFixture setup3; setup3.InitSource(topicName); - setup3.LoadSource(state2); - - auto result = setup3.SourceReadUntil<TString>(UVParser, 1); - std::vector<TString> data {"futherData"}; - UNIT_ASSERT_EQUAL(result, data); - - // pq session is steel alive - - PQWrite({"yetAnotherData"}, topicName); - - auto checkpoint = CreateCheckpoint(); + setup3.LoadSource(state2); + + auto result = setup3.SourceReadUntil<TString>(UVParser, 1); + std::vector<TString> data {"futherData"}; + UNIT_ASSERT_EQUAL(result, data); + + // pq session is steel alive + + PQWrite({"yetAnotherData"}, topicName); + + auto checkpoint = CreateCheckpoint(); setup3.SaveSourceState(checkpoint, state3); - } - - // Load the first state and check it. - { + } + + // Load the first state and check it. + { TPqIoTestFixture setup4; setup4.InitSource(topicName); - setup4.LoadSource(state); - - auto result = setup4.SourceReadUntil<TString>(UVParser, 3); - std::vector<TString> data {"data", "futherData", "yetAnotherData"}; - UNIT_ASSERT_EQUAL(result, data); - } - - // Load graphState2 and check it (offsets were saved). - { + setup4.LoadSource(state); + + auto result = setup4.SourceReadUntil<TString>(UVParser, 3); + std::vector<TString> data {"data", "futherData", "yetAnotherData"}; + UNIT_ASSERT_EQUAL(result, data); + } + + // Load graphState2 and check it (offsets were saved). + { TPqIoTestFixture setup5; setup5.InitSource(topicName); - setup5.LoadSource(state2); - - auto result = setup5.SourceReadUntil<TString>(UVParser, 2); - std::vector<TString> data {"futherData", "yetAnotherData"}; - UNIT_ASSERT_EQUAL(result, data); - } - - // Load graphState3 and check it (other offsets). - { + setup5.LoadSource(state2); + + auto result = setup5.SourceReadUntil<TString>(UVParser, 2); + std::vector<TString> data {"futherData", "yetAnotherData"}; + UNIT_ASSERT_EQUAL(result, data); + } + + // Load graphState3 and check it (other offsets). + { TPqIoTestFixture setup6; setup6.InitSource(topicName); - setup6.LoadSource(state3); - - auto result = setup6.SourceReadUntil<TString>(UVParser, 1); - std::vector<TString> data {"yetAnotherData"}; - UNIT_ASSERT_EQUAL(result, data); - } - } - - Y_UNIT_TEST(LoadCorruptedState) { + setup6.LoadSource(state3); + + auto result = setup6.SourceReadUntil<TString>(UVParser, 1); + std::vector<TString> data {"yetAnotherData"}; + UNIT_ASSERT_EQUAL(result, data); + } + } + + Y_UNIT_TEST(LoadCorruptedState) { NDqProto::TSourceState state; - const TString topicName = "Invalid"; // We wouldn't read from this topic. - auto checkpoint = CreateCheckpoint(); - - { + const TString topicName = "Invalid"; // We wouldn't read from this topic. + auto checkpoint = CreateCheckpoint(); + + { TPqIoTestFixture setup1; setup1.InitSource(topicName); setup1.SaveSourceState(checkpoint, state); - } - - // Corrupt state. + } + + // Corrupt state. TString corruptedBlob = state.GetData(0).GetStateData().GetBlob(); corruptedBlob.append('a'); state.MutableData(0)->MutableStateData()->SetBlob(corruptedBlob); - - { + + { TPqIoTestFixture setup2; setup2.InitSource(topicName); - UNIT_ASSERT_EXCEPTION_CONTAINS(setup2.LoadSource(state), yexception, "Serialized state is corrupted"); - } - } + UNIT_ASSERT_EXCEPTION_CONTAINS(setup2.LoadSource(state), yexception, "Serialized state is corrupted"); + } + } Y_UNIT_TEST(TestLoadFromSeveralStates) { const TString topicName = "LoadFromSeveralStates"; @@ -209,5 +209,5 @@ Y_UNIT_TEST_SUITE(TDqPqReadActorTest) { std::vector<TString> dataResult {"data2", "data3"}; UNIT_ASSERT_EQUAL(result, dataResult); } -} -} // namespace NKikimr::NMiniKQL +} +} // namespace NKikimr::NMiniKQL diff --git a/ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp b/ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp index 9b7fc77d92e..b64c7ee0e35 100644 --- a/ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp +++ b/ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp @@ -1,139 +1,139 @@ -#include "ut_helpers.h" - +#include "ut_helpers.h" + #include <ydb/library/yql/utils/yql_panic.h> - -#include <library/cpp/testing/unittest/registar.h> - -namespace NYql::NDq { - -constexpr TDuration WaitTimeout = TDuration::MilliSeconds(10000); - -Y_UNIT_TEST_SUITE(TPqWriterTest) { + +#include <library/cpp/testing/unittest/registar.h> + +namespace NYql::NDq { + +constexpr TDuration WaitTimeout = TDuration::MilliSeconds(10000); + +Y_UNIT_TEST_SUITE(TPqWriterTest) { Y_UNIT_TEST_F(TestWriteToTopic, TPqIoTestFixture) { - const TString topicName = "WriteToTopic"; + const TString topicName = "WriteToTopic"; InitSink(topicName); - - const std::vector<TString> data = { "1", "2", "3", "4" }; - + + const std::vector<TString> data = { "1", "2", "3", "4" }; + SinkWrite(data); - auto result = PQReadUntil(topicName, 4); - UNIT_ASSERT_EQUAL(result, data); - } - + auto result = PQReadUntil(topicName, 4); + UNIT_ASSERT_EQUAL(result, data); + } + Y_UNIT_TEST_F(TestWriteToTopicMultiBatch, TPqIoTestFixture) { - const TString topicName = "WriteToTopicMultiBatch"; + const TString topicName = "WriteToTopicMultiBatch"; InitSink(topicName); - - const std::vector<TString> data1 = { "1" }; - const std::vector<TString> data2 = { "2" }; - const std::vector<TString> data3 = { "3" }; - + + const std::vector<TString> data1 = { "1" }; + const std::vector<TString> data2 = { "2" }; + const std::vector<TString> data3 = { "3" }; + SinkWrite(data1); SinkWrite(data2); SinkWrite(data3); - auto result = PQReadUntil(topicName, 3); - - std::vector<TString> expected = { "1", "2", "3" }; - UNIT_ASSERT_EQUAL(result, expected); - } - + auto result = PQReadUntil(topicName, 3); + + std::vector<TString> expected = { "1", "2", "3" }; + UNIT_ASSERT_EQUAL(result, expected); + } + Y_UNIT_TEST_F(TestDeferredWriteToTopic, TPqIoTestFixture) { - // In this case we are checking free space overflow - const TString topicName = "DeferredWriteToTopic"; + // In this case we are checking free space overflow + const TString topicName = "DeferredWriteToTopic"; InitSink(topicName, 1); - - const std::vector<TString> data = { "1", "2", "3" }; - + + const std::vector<TString> data = { "1", "2", "3" }; + auto future = CaSetup->SinkPromises.ResumeExecution.GetFuture(); SinkWrite(data); - auto result = PQReadUntil(topicName, 3); - - UNIT_ASSERT_EQUAL(result, data); - UNIT_ASSERT(future.Wait(WaitTimeout)); // Resume execution should be called - - const std::vector<TString> data2 = { "4", "5", "6" }; - + auto result = PQReadUntil(topicName, 3); + + UNIT_ASSERT_EQUAL(result, data); + UNIT_ASSERT(future.Wait(WaitTimeout)); // Resume execution should be called + + const std::vector<TString> data2 = { "4", "5", "6" }; + SinkWrite(data2); - auto result2 = PQReadUntil(topicName, 6); - const std::vector<TString> expected = { "1", "2", "3", "4", "5", "6" }; - UNIT_ASSERT_EQUAL(result2, expected); - } - + auto result2 = PQReadUntil(topicName, 6); + const std::vector<TString> expected = { "1", "2", "3", "4", "5", "6" }; + UNIT_ASSERT_EQUAL(result2, expected); + } + Y_UNIT_TEST_F(WriteNonExistentTopic, TPqIoTestFixture) { - const TString topicName = "NonExistentTopic"; + const TString topicName = "NonExistentTopic"; InitSink(topicName); - - const std::vector<TString> data = { "1" }; + + const std::vector<TString> data = { "1" }; auto future = CaSetup->SinkPromises.Issue.GetFuture(); SinkWrite(data); - - UNIT_ASSERT(future.Wait(WaitTimeout)); + + UNIT_ASSERT(future.Wait(WaitTimeout)); UNIT_ASSERT_STRING_CONTAINS(future.GetValue().ToString(), "Write session to topic \"NonExistentTopic\" was closed"); - } - - Y_UNIT_TEST(TestCheckpoints) { - const TString topicName = "Checkpoints"; - + } + + Y_UNIT_TEST(TestCheckpoints) { + const TString topicName = "Checkpoints"; + NDqProto::TSinkState state1; - { + { TPqIoTestFixture setup; setup.InitSink(topicName); - - const std::vector<TString> data1 = { "1" }; + + const std::vector<TString> data1 = { "1" }; setup.SinkWrite(data1); - - const std::vector<TString> data2 = { "2", "3" }; - auto checkpoint = CreateCheckpoint(); + + const std::vector<TString> data2 = { "2", "3" }; + auto checkpoint = CreateCheckpoint(); auto future = setup.CaSetup->SinkPromises.StateSaved.GetFuture(); setup.SinkWrite(data2, checkpoint); - - UNIT_ASSERT(future.Wait(WaitTimeout)); - state1 = future.GetValue(); - } - - { + + UNIT_ASSERT(future.Wait(WaitTimeout)); + state1 = future.GetValue(); + } + + { TPqIoTestFixture setup; setup.InitSink(topicName); - setup.LoadSink(state1); - - const std::vector<TString> data3 = { "4", "5" }; + setup.LoadSink(state1); + + const std::vector<TString> data3 = { "4", "5" }; setup.SinkWrite(data3); - - auto result = PQReadUntil(topicName, 5); - const std::vector<TString> expected = { "1", "2", "3", "4", "5" }; - UNIT_ASSERT_EQUAL(result, expected); - } - - { + + auto result = PQReadUntil(topicName, 5); + const std::vector<TString> expected = { "1", "2", "3", "4", "5" }; + UNIT_ASSERT_EQUAL(result, expected); + } + + { TPqIoTestFixture setup; setup.InitSink(topicName); - setup.LoadSink(state1); - - const std::vector<TString> data4 = { "4", "5" }; + setup.LoadSink(state1); + + const std::vector<TString> data4 = { "4", "5" }; setup.SinkWrite(data4); // This write should be deduplicated - - auto result = PQReadUntil(topicName, 4); - const std::vector<TString> expected = { "1", "2", "3", "4", "5" }; - UNIT_ASSERT_EQUAL(result, expected); - } - } - + + auto result = PQReadUntil(topicName, 4); + const std::vector<TString> expected = { "1", "2", "3", "4", "5" }; + UNIT_ASSERT_EQUAL(result, expected); + } + } + Y_UNIT_TEST_F(TestCheckpointWithEmptyBatch, TPqIoTestFixture) { - const TString topicName = "Checkpoints"; - + const TString topicName = "Checkpoints"; + NDqProto::TSinkState state1; - { + { InitSink(topicName); - - const std::vector<TString> data = {}; - auto checkpoint = CreateCheckpoint(); + + const std::vector<TString> data = {}; + auto checkpoint = CreateCheckpoint(); auto future = CaSetup->SinkPromises.StateSaved.GetFuture(); SinkWrite(data, checkpoint); - - UNIT_ASSERT(future.Wait(WaitTimeout)); - state1 = future.GetValue(); - } - } -} - -} // NYql::NDq + + UNIT_ASSERT(future.Wait(WaitTimeout)); + state1 = future.GetValue(); + } + } +} + +} // NYql::NDq diff --git a/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp b/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp index 37d8af92eab..6fcae4e0a7c 100644 --- a/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp +++ b/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp @@ -1,37 +1,37 @@ -#include "ut_helpers.h" - +#include "ut_helpers.h" + #include <ydb/library/yql/minikql/mkql_string_util.h> - + #include <ydb/core/testlib/basics/appdata.h> - -#include <util/system/env.h> - -#include <condition_variable> -#include <thread> - -namespace NYql::NDq { - -using namespace NActors; - -NYql::NPq::NProto::TDqPqTopicSource BuildPqTopicSourceSettings(TString topic) { - NYql::NPq::NProto::TDqPqTopicSource settings; - settings.SetTopicPath(topic); - settings.SetConsumerName(DefaultPqConsumer); - settings.SetEndpoint(GetDefaultPqEndpoint()); - settings.MutableToken()->SetName("token"); - - return settings; -} - -NYql::NPq::NProto::TDqPqTopicSink BuildPqTopicSinkSettings(TString topic) { - NYql::NPq::NProto::TDqPqTopicSink settings; - settings.SetTopicPath(topic); - settings.SetEndpoint(GetDefaultPqEndpoint()); - settings.MutableToken()->SetName("token"); - - return settings; -} - + +#include <util/system/env.h> + +#include <condition_variable> +#include <thread> + +namespace NYql::NDq { + +using namespace NActors; + +NYql::NPq::NProto::TDqPqTopicSource BuildPqTopicSourceSettings(TString topic) { + NYql::NPq::NProto::TDqPqTopicSource settings; + settings.SetTopicPath(topic); + settings.SetConsumerName(DefaultPqConsumer); + settings.SetEndpoint(GetDefaultPqEndpoint()); + settings.MutableToken()->SetName("token"); + + return settings; +} + +NYql::NPq::NProto::TDqPqTopicSink BuildPqTopicSinkSettings(TString topic) { + NYql::NPq::NProto::TDqPqTopicSink settings; + settings.SetTopicPath(topic); + settings.SetEndpoint(GetDefaultPqEndpoint()); + settings.MutableToken()->SetName("token"); + + return settings; +} + TPqIoTestFixture::TPqIoTestFixture() { } @@ -41,144 +41,144 @@ TPqIoTestFixture::~TPqIoTestFixture() { } void TPqIoTestFixture::InitSource( - NYql::NPq::NProto::TDqPqTopicSource&& settings, - i64 freeSpace) -{ + NYql::NPq::NProto::TDqPqTopicSource&& settings, + i64 freeSpace) +{ CaSetup->Execute([&](TFakeActor& actor) { - NPq::NProto::TDqReadTaskParams params; - auto* partitioninigParams = params.MutablePartitioningParams(); - partitioninigParams->SetTopicPartitionsCount(1); - partitioninigParams->SetEachTopicPartitionGroupId(0); - partitioninigParams->SetDqPartitionsCount(1); - - TString serializedParams; + NPq::NProto::TDqReadTaskParams params; + auto* partitioninigParams = params.MutablePartitioningParams(); + partitioninigParams->SetTopicPartitionsCount(1); + partitioninigParams->SetEachTopicPartitionGroupId(0); + partitioninigParams->SetDqPartitionsCount(1); + + TString serializedParams; Y_PROTOBUF_SUPPRESS_NODISCARD params.SerializeToString(&serializedParams); - - const THashMap<TString, TString> secureParams; - const THashMap<TString, TString> taskParams { {"pq", serializedParams} }; - - auto [dqSource, dqSourceAsActor] = CreateDqPqReadActor( - std::move(settings), - 0, - "query_1", - secureParams, - taskParams, + + const THashMap<TString, TString> secureParams; + const THashMap<TString, TString> taskParams { {"pq", serializedParams} }; + + auto [dqSource, dqSourceAsActor] = CreateDqPqReadActor( + std::move(settings), + 0, + "query_1", + secureParams, + taskParams, Driver, - nullptr, - &actor.GetSourceCallbacks(), - actor.GetHolderFactory(), - freeSpace); - - actor.InitSource(dqSource, dqSourceAsActor); - }); -} - + nullptr, + &actor.GetSourceCallbacks(), + actor.GetHolderFactory(), + freeSpace); + + actor.InitSource(dqSource, dqSourceAsActor); + }); +} + void TPqIoTestFixture::InitSink( - NPq::NProto::TDqPqTopicSink&& settings, - i64 freeSpace) -{ - const THashMap<TString, TString> secureParams; - + NPq::NProto::TDqPqTopicSink&& settings, + i64 freeSpace) +{ + const THashMap<TString, TString> secureParams; + CaSetup->Execute([&](TFakeActor& actor) { - auto [dqSink, dqSinkAsActor] = CreateDqPqWriteActor( - std::move(settings), - 0, - "query_1", - secureParams, + auto [dqSink, dqSinkAsActor] = CreateDqPqWriteActor( + std::move(settings), + 0, + "query_1", + secureParams, Driver, - nullptr, - &actor.GetSinkCallbacks(), - freeSpace); - - actor.InitSink(dqSink, dqSinkAsActor); - }); -} - -TString GetDefaultPqEndpoint() { - auto port = GetEnv("LOGBROKER_PORT"); - UNIT_ASSERT_C(port, "Logbroker recipe is expected"); - return TStringBuilder() << "localhost:" << port; -} - -extern const TString DefaultPqConsumer = "test_client"; - -void PQWrite( - const std::vector<TString>& sequence, - const TString& topic, - const TString& endpoint) -{ - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(endpoint); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr")); - NYdb::TDriver driver(cfg); - NYdb::NPersQueue::TPersQueueClient client(driver); - NYdb::NPersQueue::TWriteSessionSettings sessionSettings; - sessionSettings - .Path(topic) - .MessageGroupId("src_id"); - auto session = client.CreateSimpleBlockingWriteSession(sessionSettings); - for (const TString& data : sequence) { + nullptr, + &actor.GetSinkCallbacks(), + freeSpace); + + actor.InitSink(dqSink, dqSinkAsActor); + }); +} + +TString GetDefaultPqEndpoint() { + auto port = GetEnv("LOGBROKER_PORT"); + UNIT_ASSERT_C(port, "Logbroker recipe is expected"); + return TStringBuilder() << "localhost:" << port; +} + +extern const TString DefaultPqConsumer = "test_client"; + +void PQWrite( + const std::vector<TString>& sequence, + const TString& topic, + const TString& endpoint) +{ + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(endpoint); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr")); + NYdb::TDriver driver(cfg); + NYdb::NPersQueue::TPersQueueClient client(driver); + NYdb::NPersQueue::TWriteSessionSettings sessionSettings; + sessionSettings + .Path(topic) + .MessageGroupId("src_id"); + auto session = client.CreateSimpleBlockingWriteSession(sessionSettings); + for (const TString& data : sequence) { UNIT_ASSERT_C(session->Write(data), "Failed to write message with body \"" << data << "\" to topic " << topic); - Cerr << "Message '" << data << "' was written into topic '" << topic << "'" << Endl; - } - session->Close(); // Wait until all data would be written into PQ. - session = nullptr; - driver.Stop(true); -} - -std::vector<TString> PQReadUntil( - const TString& topic, - ui64 size, - const TString& endpoint, - TDuration timeout) -{ - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(endpoint); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr")); - NYdb::TDriver driver(cfg); - NYdb::NPersQueue::TPersQueueClient client(driver); - NYdb::NPersQueue::TReadSessionSettings sessionSettings; - sessionSettings - .AppendTopics(topic) - .ConsumerName(DefaultPqConsumer); - - auto promise = NThreading::NewPromise(); - std::vector<TString> result; - - sessionSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) { - for (const auto& message : ev.GetMessages()) { - result.emplace_back(message.GetData()); - } - if (result.size() >= size) { - promise.SetValue(); - } - }, false, false); - - std::shared_ptr<NYdb::NPersQueue::IReadSession> session = client.CreateReadSession(sessionSettings); - UNIT_ASSERT(promise.GetFuture().Wait(timeout)); - session->Close(TDuration::Zero()); - session = nullptr; - driver.Stop(true); - return result; -} - -std::vector<TString> UVParser(const NUdf::TUnboxedValue& item) { - return { TString(item.AsStringRef()) }; -} - + Cerr << "Message '" << data << "' was written into topic '" << topic << "'" << Endl; + } + session->Close(); // Wait until all data would be written into PQ. + session = nullptr; + driver.Stop(true); +} + +std::vector<TString> PQReadUntil( + const TString& topic, + ui64 size, + const TString& endpoint, + TDuration timeout) +{ + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(endpoint); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr")); + NYdb::TDriver driver(cfg); + NYdb::NPersQueue::TPersQueueClient client(driver); + NYdb::NPersQueue::TReadSessionSettings sessionSettings; + sessionSettings + .AppendTopics(topic) + .ConsumerName(DefaultPqConsumer); + + auto promise = NThreading::NewPromise(); + std::vector<TString> result; + + sessionSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) { + for (const auto& message : ev.GetMessages()) { + result.emplace_back(message.GetData()); + } + if (result.size() >= size) { + promise.SetValue(); + } + }, false, false); + + std::shared_ptr<NYdb::NPersQueue::IReadSession> session = client.CreateReadSession(sessionSettings); + UNIT_ASSERT(promise.GetFuture().Wait(timeout)); + session->Close(TDuration::Zero()); + session = nullptr; + driver.Stop(true); + return result; +} + +std::vector<TString> UVParser(const NUdf::TUnboxedValue& item) { + return { TString(item.AsStringRef()) }; +} + void TPqIoTestFixture::SinkWrite(std::vector<TString> data, TMaybe<NDqProto::TCheckpoint> checkpoint) { CaSetup->SinkWrite([data](NKikimr::NMiniKQL::THolderFactory& factory) { - NKikimr::NMiniKQL::TUnboxedValueVector batch; - batch.reserve(data.size()); - for (const auto& item : data) { + NKikimr::NMiniKQL::TUnboxedValueVector batch; + batch.reserve(data.size()); + for (const auto& item : data) { NUdf::TUnboxedValue* unboxedValueForData = nullptr; - batch.emplace_back(factory.CreateDirectArrayHolder(1, unboxedValueForData)); + batch.emplace_back(factory.CreateDirectArrayHolder(1, unboxedValueForData)); unboxedValueForData[0] = NKikimr::NMiniKQL::MakeString(NUdf::TStringRef(item.Data(), item.Size())); - } - - return batch; - }, checkpoint); -} -} + } + + return batch; + }, checkpoint); +} +} diff --git a/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h b/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h index 59cd28f7170..5d9f3d11424 100644 --- a/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h +++ b/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h> #include <ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h> #include <ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h> @@ -8,28 +8,28 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> #include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> #include <ydb/library/yql/minikql/mkql_alloc.h> - + #include <ydb/public/sdk/cpp/client/ydb_persqueue_public/persqueue.h> #include <ydb/core/testlib/basics/runtime.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <chrono> -#include <queue> - -namespace NYql::NDq { - -NYql::NPq::NProto::TDqPqTopicSource BuildPqTopicSourceSettings(TString topic); - -NYql::NPq::NProto::TDqPqTopicSink BuildPqTopicSinkSettings(TString topic); - + +#include <library/cpp/testing/unittest/registar.h> + +#include <chrono> +#include <queue> + +namespace NYql::NDq { + +NYql::NPq::NProto::TDqPqTopicSource BuildPqTopicSourceSettings(TString topic); + +NYql::NPq::NProto::TDqPqTopicSink BuildPqTopicSinkSettings(TString topic); + struct TPqIoTestFixture : public NUnitTest::TBaseFixture { std::unique_ptr<TFakeCASetup> CaSetup = std::make_unique<TFakeCASetup>(); NYdb::TDriver Driver = NYdb::TDriver(NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr"))); - + TPqIoTestFixture(); ~TPqIoTestFixture(); - + void InitSource( NYql::NPq::NProto::TDqPqTopicSource&& settings, i64 freeSpace = 1_MB); @@ -83,24 +83,24 @@ struct TPqIoTestFixture : public NUnitTest::TBaseFixture { void SinkWrite(std::vector<TString> data, TMaybe<NDqProto::TCheckpoint> checkpoint = Nothing()); }; -TString GetDefaultPqEndpoint(); - -extern const TString DefaultPqConsumer; -extern const TString DefaultPqCluster; - -// Write using YDB driver -void PQWrite( - const std::vector<TString>& sequence, - const TString& topic, - const TString& endpoint = GetDefaultPqEndpoint()); - -// Read using YDB driver -std::vector<TString> PQReadUntil( - const TString& topic, - ui64 size, - const TString& endpoint = GetDefaultPqEndpoint(), - TDuration timeout = TDuration::MilliSeconds(10000)); - -std::vector<TString> UVParser(const NUdf::TUnboxedValue& item); - -} +TString GetDefaultPqEndpoint(); + +extern const TString DefaultPqConsumer; +extern const TString DefaultPqCluster; + +// Write using YDB driver +void PQWrite( + const std::vector<TString>& sequence, + const TString& topic, + const TString& endpoint = GetDefaultPqEndpoint()); + +// Read using YDB driver +std::vector<TString> PQReadUntil( + const TString& topic, + ui64 size, + const TString& endpoint = GetDefaultPqEndpoint(), + TDuration timeout = TDuration::MilliSeconds(10000)); + +std::vector<TString> UVParser(const NUdf::TUnboxedValue& item); + +} diff --git a/ydb/library/yql/providers/pq/async_io/ut/ya.make b/ydb/library/yql/providers/pq/async_io/ut/ya.make index a6bcd4a304b..1ce7d821550 100644 --- a/ydb/library/yql/providers/pq/async_io/ut/ya.make +++ b/ydb/library/yql/providers/pq/async_io/ut/ya.make @@ -1,26 +1,26 @@ UNITTEST_FOR(ydb/library/yql/providers/pq/async_io) - -OWNER( - d-mokhnatkin + +OWNER( + d-mokhnatkin g:yq - g:yql -) - -ENV(YDB_USE_IN_MEMORY_PDISKS=true) + g:yql +) + +ENV(YDB_USE_IN_MEMORY_PDISKS=true) ENV(LOGBROKER_CREATE_TOPICS=ReadFromTopic,ReadWithFreeSpace,SaveLoadPqRead,WriteToTopic,WriteToTopicMultiBatch,DeferredWriteToTopic,SaveLoadPqWrite,Checkpoints,LoadFromSeveralStates) -ENV(LOGBROKER_TOPICS_PARTITIONS=1) +ENV(LOGBROKER_TOPICS_PARTITIONS=1) INCLUDE(${ARCADIA_ROOT}/kikimr/yq/tools/lbk_recipe_with_dummy_cm/recipe.inc) - -SRCS( - dq_pq_read_actor_ut.cpp - dq_pq_write_actor_ut.cpp - ut_helpers.cpp -) - -PEERDIR( + +SRCS( + dq_pq_read_actor_ut.cpp + dq_pq_write_actor_ut.cpp + ut_helpers.cpp +) + +PEERDIR( ydb/core/testlib/basics ydb/library/yql/minikql ydb/library/yql/minikql/computation @@ -29,8 +29,8 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_persqueue_public ydb/library/yql/providers/common/comp_nodes ydb/library/yql/providers/common/ut_helpers -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/pq/async_io/ya.make b/ydb/library/yql/providers/pq/async_io/ya.make index cf5ee619971..8c37ecec6fa 100644 --- a/ydb/library/yql/providers/pq/async_io/ya.make +++ b/ydb/library/yql/providers/pq/async_io/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - d-mokhnatkin +LIBRARY() + +OWNER( + d-mokhnatkin g:yq - g:yql -) - -SRCS( - dq_pq_read_actor.cpp - dq_pq_write_actor.cpp - probes.cpp -) - -PEERDIR( + g:yql +) + +SRCS( + dq_pq_read_actor.cpp + dq_pq_write_actor.cpp + probes.cpp +) + +PEERDIR( ydb/library/yql/minikql/computation ydb/library/yql/providers/common/token_accessor/client ydb/library/yql/public/types @@ -22,12 +22,12 @@ PEERDIR( ydb/public/sdk/cpp/client/ydb_types/credentials ydb/library/yql/dq/actors/compute ydb/library/yql/providers/pq/proto -) - -YQL_LAST_ABI_VERSION() - -END() - +) + +YQL_LAST_ABI_VERSION() + +END() + IF (NOT OPENSOURCE) IF (OS_LINUX) # Logbroker recipe is supported only for linux. diff --git a/ydb/library/yql/providers/pq/cm_client/lib/client.cpp b/ydb/library/yql/providers/pq/cm_client/lib/client.cpp index 0391d1af902..71d878f5e57 100644 --- a/ydb/library/yql/providers/pq/cm_client/lib/client.cpp +++ b/ydb/library/yql/providers/pq/cm_client/lib/client.cpp @@ -246,7 +246,7 @@ private: NGrpc::TGRpcClientConfig MakeConnectionConfig() const { NGrpc::TGRpcClientConfig cfg(Options.GetEndpoint(), Options.GetRequestTimeout()); cfg.CompressionAlgoritm = GRPC_COMPRESS_GZIP; - cfg.EnableSsl = Options.GetEnableSsl(); + cfg.EnableSsl = Options.GetEnableSsl(); return cfg; } diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp index e2b1e741a8b..cd536762e4a 100644 --- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp +++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp @@ -9,8 +9,8 @@ NPq::NConfigurationManager::TClientOptions GetCmClientOptions(const NYql::TPqClu NPq::NConfigurationManager::TClientOptions opts; opts .SetEndpoint(cfg.GetConfigManagerEndpoint()) - .SetCredentialsProviderFactory(credentialsProviderFactory) - .SetEnableSsl(cfg.GetUseSsl()); + .SetCredentialsProviderFactory(credentialsProviderFactory) + .SetEnableSsl(cfg.GetUseSsl()); return opts; } @@ -19,7 +19,7 @@ NYdb::NPersQueue::TPersQueueClientSettings GetYdbPqClientOptions(const TString& NYdb::NPersQueue::TPersQueueClientSettings opts; opts .DiscoveryEndpoint(cfg.GetEndpoint()) - .Database(database) + .Database(database) .EnableSsl(cfg.GetUseSsl()) .CredentialsProviderFactory(credentialsProviderFactory); diff --git a/ydb/library/yql/providers/pq/proto/dq_io.proto b/ydb/library/yql/providers/pq/proto/dq_io.proto index 4f86385353c..e87eb72755c 100644 --- a/ydb/library/yql/providers/pq/proto/dq_io.proto +++ b/ydb/library/yql/providers/pq/proto/dq_io.proto @@ -1,37 +1,37 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package NYql.NPq.NProto; - -message TToken { - string Name = 1; // Key in secure params. - // TODO: token type (oauth, tvm, iam etc). -} - -enum EClusterType { - Unspecified = 0; - PersQueue = 1; - DataStreams = 2; -} - -message TDqPqTopicSource { - string TopicPath = 1; - string ConsumerName = 2; - string Endpoint = 3; - TToken Token = 4; - string Database = 5; - EClusterType ClusterType = 6; +syntax = "proto3"; +option cc_enable_arenas = true; + +package NYql.NPq.NProto; + +message TToken { + string Name = 1; // Key in secure params. + // TODO: token type (oauth, tvm, iam etc). +} + +enum EClusterType { + Unspecified = 0; + PersQueue = 1; + DataStreams = 2; +} + +message TDqPqTopicSource { + string TopicPath = 1; + string ConsumerName = 2; + string Endpoint = 3; + TToken Token = 4; + string Database = 5; + EClusterType ClusterType = 6; bool UseSsl = 7; bool AddBearerToToken = 8; string DatabaseId = 9; -} - -message TDqPqTopicSink { - string TopicPath = 1; - string Endpoint = 2; - TToken Token = 3; - string Database = 4; +} + +message TDqPqTopicSink { + string TopicPath = 1; + string Endpoint = 2; + TToken Token = 3; + string Database = 4; bool UseSsl = 5; bool AddBearerToToken = 6; - EClusterType ClusterType = 7; -} + EClusterType ClusterType = 7; +} diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp index ced4c9dc98e..ea67ffd422c 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp @@ -131,14 +131,14 @@ public: return {}; } - static NPq::NProto::EClusterType ToClusterType(NYql::TPqClusterConfig::EClusterType t) { + static NPq::NProto::EClusterType ToClusterType(NYql::TPqClusterConfig::EClusterType t) { switch (t) { case NYql::TPqClusterConfig::CT_UNSPECIFIED: - return NPq::NProto::Unspecified; + return NPq::NProto::Unspecified; case NYql::TPqClusterConfig::CT_PERS_QUEUE: - return NPq::NProto::PersQueue; + return NPq::NProto::PersQueue; case NYql::TPqClusterConfig::CT_DATA_STREAMS: - return NPq::NProto::DataStreams; + return NPq::NProto::DataStreams; } } diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp index 753987fe6a9..8886ddb83e7 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp @@ -13,17 +13,17 @@ void RegisterDqPqMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler) { compiler.ChainCallable(TDqSourceWideWrap::CallableName(), [](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) { if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == PqProviderName) { - const auto wrapped = TryWrapWithParser(wrapper, ctx); - if (wrapped) { - return *wrapped; + const auto wrapped = TryWrapWithParser(wrapper, ctx); + if (wrapped) { + return *wrapped; } - const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx); + const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx); auto flow = ctx.ProgramBuilder.ToFlow(input); return ctx.ProgramBuilder.ExpandMap(flow, - [&](TRuntimeNode item) -> TRuntimeNode::TList { - return {item}; - }); + [&](TRuntimeNode item) -> TRuntimeNode::TList { + return {item}; + }); } return TRuntimeNode(); diff --git a/ydb/library/yql/providers/pq/ya.make b/ydb/library/yql/providers/pq/ya.make index f093910cc75..840d37c7d53 100644 --- a/ydb/library/yql/providers/pq/ya.make +++ b/ydb/library/yql/providers/pq/ya.make @@ -5,7 +5,7 @@ OWNER( ) RECURSE( - async_io + async_io common cm_client expr_nodes diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp index 0b25623b96c..eb5410a24e3 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp +++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp @@ -190,7 +190,7 @@ private: return; } ++IsDoneCounter; - Callbacks->OnSourceError(InputIndex, result->Get()->Error, true); + Callbacks->OnSourceError(InputIndex, result->Get()->Error, true); } // IActor & IDqSourceActor diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp index 5a2f3e100c8..fb782404982 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp +++ b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp @@ -2,7 +2,7 @@ #include "yql_s3_read_actor.h" #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> - + namespace NYql::NDq { void RegisterS3ReadActorFactory( diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp index cb2822fb765..fab2d6d0f75 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp @@ -14,10 +14,10 @@ void RegisterDqS3MkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, con compiler.ChainCallable(TDqSourceWideWrap::CallableName(), [](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) { if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == S3ProviderName) { - const auto wrapped = TryWrapWithParser(wrapper, ctx); - if (wrapped) { - return *wrapped; - } + const auto wrapped = TryWrapWithParser(wrapper, ctx); + if (wrapped) { + return *wrapped; + } } return TRuntimeNode(); diff --git a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp index fcb919a59e6..fc07635176f 100644 --- a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp +++ b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp @@ -1,10 +1,10 @@ -#include "dq_solomon_write_actor.h" -#include "metrics_encoder.h" - +#include "dq_solomon_write_actor.h" +#include "metrics_encoder.h" + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> #include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> #include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h> - + #include <ydb/library/yql/utils/actor_log/log.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h> @@ -12,472 +12,472 @@ #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/utils/yql_panic.h> #include <ydb/library/yql/utils/actors/http_sender_actor.h> - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> -#include <library/cpp/actors/http/http_proxy.h> -#include <library/cpp/json/easy_parse/json_easy_parser.h> - - -#include <util/generic/algorithm.h> -#include <util/generic/hash.h> -#include <util/system/compiler.h> - -#include <algorithm> -#include <queue> -#include <variant> - -#define SINK_LOG_T(s) \ - LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) -#define SINK_LOG_D(s) \ - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) -#define SINK_LOG_I(s) \ - LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) -#define SINK_LOG_W(s) \ - LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) -#define SINK_LOG_N(s) \ - LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) -#define SINK_LOG_E(s) \ - LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) -#define SINK_LOG_C(s) \ - LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) -#define SINK_LOG(prio, s) \ - LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) - -namespace NYql::NDq { - -using namespace NActors; -using namespace NLog; -using namespace NKikimr::NMiniKQL; - -namespace { - -const ui64 MaxMetricsPerRequest = 1000; // Max allowed count is 10000 -const ui64 MaxRequestsInflight = 3; - -struct TDqSolomonWriteParams { - NSo::NProto::TDqSolomonShard Shard; -}; - -struct TMetricsToSend { - TString Data; - ui64 MetricsCount = 0; -}; - -struct TMetricsInflight { - TActorId HttpSenderId; - ui64 MetricsCount = 0; -}; - -} // namespace - -class TDqSolomonWriteActor : public NActors::TActor<TDqSolomonWriteActor>, public IDqSinkActor { -public: + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> +#include <library/cpp/actors/http/http_proxy.h> +#include <library/cpp/json/easy_parse/json_easy_parser.h> + + +#include <util/generic/algorithm.h> +#include <util/generic/hash.h> +#include <util/system/compiler.h> + +#include <algorithm> +#include <queue> +#include <variant> + +#define SINK_LOG_T(s) \ + LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) +#define SINK_LOG_D(s) \ + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) +#define SINK_LOG_I(s) \ + LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) +#define SINK_LOG_W(s) \ + LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) +#define SINK_LOG_N(s) \ + LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) +#define SINK_LOG_E(s) \ + LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) +#define SINK_LOG_C(s) \ + LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) +#define SINK_LOG(prio, s) \ + LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s) + +namespace NYql::NDq { + +using namespace NActors; +using namespace NLog; +using namespace NKikimr::NMiniKQL; + +namespace { + +const ui64 MaxMetricsPerRequest = 1000; // Max allowed count is 10000 +const ui64 MaxRequestsInflight = 3; + +struct TDqSolomonWriteParams { + NSo::NProto::TDqSolomonShard Shard; +}; + +struct TMetricsToSend { + TString Data; + ui64 MetricsCount = 0; +}; + +struct TMetricsInflight { + TActorId HttpSenderId; + ui64 MetricsCount = 0; +}; + +} // namespace + +class TDqSolomonWriteActor : public NActors::TActor<TDqSolomonWriteActor>, public IDqSinkActor { +public: static constexpr char ActorName[] = "DQ_SOLOMON_WRITE_ACTOR"; - TDqSolomonWriteActor( - ui64 outputIndex, - TDqSolomonWriteParams&& writeParams, - NYql::NDq::IDqSinkActor::ICallbacks* callbacks, + TDqSolomonWriteActor( + ui64 outputIndex, + TDqSolomonWriteParams&& writeParams, + NYql::NDq::IDqSinkActor::ICallbacks* callbacks, const NMonitoring::TDynamicCounterPtr& counters, std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider, - i64 freeSpace) - : TActor<TDqSolomonWriteActor>(&TDqSolomonWriteActor::StateFunc) - , OutputIndex(outputIndex) - , WriteParams(std::move(writeParams)) - , Url(GetUrl()) - , Callbacks(callbacks) - , Metrics(counters) - , FreeSpace(freeSpace) - , UserMetricsEncoder( - WriteParams.Shard.GetScheme(), - WriteParams.Shard.GetClusterType() == NSo::NProto::ESolomonClusterType::CT_MONITORING) + i64 freeSpace) + : TActor<TDqSolomonWriteActor>(&TDqSolomonWriteActor::StateFunc) + , OutputIndex(outputIndex) + , WriteParams(std::move(writeParams)) + , Url(GetUrl()) + , Callbacks(callbacks) + , Metrics(counters) + , FreeSpace(freeSpace) + , UserMetricsEncoder( + WriteParams.Shard.GetScheme(), + WriteParams.Shard.GetClusterType() == NSo::NProto::ESolomonClusterType::CT_MONITORING) , CredentialsProvider(credentialsProvider) - { - SINK_LOG_D("Init"); - } - - STRICT_STFUNC(StateFunc, + { + SINK_LOG_D("Init"); + } + + STRICT_STFUNC(StateFunc, hFunc(TEvHttpBase::TEvSendResult, Handle); - ) - -public: - void SendData( - NKikimr::NMiniKQL::TUnboxedValueVector&& batch, - i64, - const TMaybe<NDqProto::TCheckpoint>& checkpoint, - bool) override - { - SINK_LOG_D("Got " << batch.size() << " items to send"); - - ui64 metricsCount = 0; - for (const auto& item : batch) { - if (metricsCount + WriteParams.Shard.GetScheme().GetSensors().size() > MaxMetricsPerRequest) { - PushMetricsToBuffer(metricsCount); - } - - metricsCount += UserMetricsEncoder.Append(item); - } - - if (metricsCount != 0) { - PushMetricsToBuffer(metricsCount); - } - - if (checkpoint) { - SendingBuffer.emplace(*checkpoint); - } - - while (TryToSendNextBatch()) {} - - if (FreeSpace <= 0) { - ShouldNotifyNewFreeSpace = true; - } - }; - + ) + +public: + void SendData( + NKikimr::NMiniKQL::TUnboxedValueVector&& batch, + i64, + const TMaybe<NDqProto::TCheckpoint>& checkpoint, + bool) override + { + SINK_LOG_D("Got " << batch.size() << " items to send"); + + ui64 metricsCount = 0; + for (const auto& item : batch) { + if (metricsCount + WriteParams.Shard.GetScheme().GetSensors().size() > MaxMetricsPerRequest) { + PushMetricsToBuffer(metricsCount); + } + + metricsCount += UserMetricsEncoder.Append(item); + } + + if (metricsCount != 0) { + PushMetricsToBuffer(metricsCount); + } + + if (checkpoint) { + SendingBuffer.emplace(*checkpoint); + } + + while (TryToSendNextBatch()) {} + + if (FreeSpace <= 0) { + ShouldNotifyNewFreeSpace = true; + } + }; + void LoadState(const NDqProto::TSinkState&) override { } - - void CommitState(const NDqProto::TCheckpoint&) override { } - - i64 GetFreeSpace() const override { - return FreeSpace; - }; - - ui64 GetOutputIndex() const override { - return OutputIndex; - }; - -private: - struct TDqSolomonWriteActorMetrics { - explicit TDqSolomonWriteActorMetrics(const NMonitoring::TDynamicCounterPtr& counters) { - auto subgroup = counters->GetSubgroup("subsystem", "dq_solomon_write_actor"); - SendingBufferSize = subgroup->GetCounter("SendingBufferSize"); - WindowMinSendingBufferSize = subgroup->GetCounter("WindowMinSendingBufferSize"); - InflightRequests = subgroup->GetCounter("InflightRequests"); - WindowMinInflightRequests = subgroup->GetCounter("WindowMinInflightRequests"); - SentMetrics = subgroup->GetCounter("SentMetrics", true); - СonfirmedMetrics = subgroup->GetCounter("СonfirmedMetrics", true); - } - - NMonitoring::TDynamicCounters::TCounterPtr SendingBufferSize; - NMonitoring::TDynamicCounters::TCounterPtr WindowMinSendingBufferSize; - NMonitoring::TDynamicCounters::TCounterPtr InflightRequests; - NMonitoring::TDynamicCounters::TCounterPtr WindowMinInflightRequests; - NMonitoring::TDynamicCounters::TCounterPtr SentMetrics; - NMonitoring::TDynamicCounters::TCounterPtr СonfirmedMetrics; - - public: - void ReportSendingBufferSize(size_t size) { - SendingBufferSizeHistoryMin += size; - SendingBufferSizeHistory.push(size); - - if (SendingBufferSizeHistory.size() > WindowSize) { - SendingBufferSizeHistoryMin -= SendingBufferSizeHistory.front(); - SendingBufferSizeHistory.pop(); - } - - *SendingBufferSize = size; - *WindowMinSendingBufferSize = SendingBufferSizeHistoryMin / SendingBufferSizeHistory.size(); - } - - void ReportInflightRequestsCount(size_t count) { - InflightRequestsHistoryMin += count; - InflightRequestsHistory.push(count); - - if (InflightRequestsHistory.size() > WindowSize) { - InflightRequestsHistoryMin -= InflightRequestsHistory.front(); - InflightRequestsHistory.pop(); - } - - *InflightRequests = count; - *WindowMinInflightRequests = InflightRequestsHistoryMin / InflightRequestsHistory.size(); - } - - private: - constexpr static size_t WindowSize = 30; - - std::queue<size_t> SendingBufferSizeHistory; - size_t SendingBufferSizeHistoryMin = 0; - - std::queue<size_t> InflightRequestsHistory; - size_t InflightRequestsHistoryMin = 0; - }; - + + void CommitState(const NDqProto::TCheckpoint&) override { } + + i64 GetFreeSpace() const override { + return FreeSpace; + }; + + ui64 GetOutputIndex() const override { + return OutputIndex; + }; + +private: + struct TDqSolomonWriteActorMetrics { + explicit TDqSolomonWriteActorMetrics(const NMonitoring::TDynamicCounterPtr& counters) { + auto subgroup = counters->GetSubgroup("subsystem", "dq_solomon_write_actor"); + SendingBufferSize = subgroup->GetCounter("SendingBufferSize"); + WindowMinSendingBufferSize = subgroup->GetCounter("WindowMinSendingBufferSize"); + InflightRequests = subgroup->GetCounter("InflightRequests"); + WindowMinInflightRequests = subgroup->GetCounter("WindowMinInflightRequests"); + SentMetrics = subgroup->GetCounter("SentMetrics", true); + СonfirmedMetrics = subgroup->GetCounter("СonfirmedMetrics", true); + } + + NMonitoring::TDynamicCounters::TCounterPtr SendingBufferSize; + NMonitoring::TDynamicCounters::TCounterPtr WindowMinSendingBufferSize; + NMonitoring::TDynamicCounters::TCounterPtr InflightRequests; + NMonitoring::TDynamicCounters::TCounterPtr WindowMinInflightRequests; + NMonitoring::TDynamicCounters::TCounterPtr SentMetrics; + NMonitoring::TDynamicCounters::TCounterPtr СonfirmedMetrics; + + public: + void ReportSendingBufferSize(size_t size) { + SendingBufferSizeHistoryMin += size; + SendingBufferSizeHistory.push(size); + + if (SendingBufferSizeHistory.size() > WindowSize) { + SendingBufferSizeHistoryMin -= SendingBufferSizeHistory.front(); + SendingBufferSizeHistory.pop(); + } + + *SendingBufferSize = size; + *WindowMinSendingBufferSize = SendingBufferSizeHistoryMin / SendingBufferSizeHistory.size(); + } + + void ReportInflightRequestsCount(size_t count) { + InflightRequestsHistoryMin += count; + InflightRequestsHistory.push(count); + + if (InflightRequestsHistory.size() > WindowSize) { + InflightRequestsHistoryMin -= InflightRequestsHistory.front(); + InflightRequestsHistory.pop(); + } + + *InflightRequests = count; + *WindowMinInflightRequests = InflightRequestsHistoryMin / InflightRequestsHistory.size(); + } + + private: + constexpr static size_t WindowSize = 30; + + std::queue<size_t> SendingBufferSizeHistory; + size_t SendingBufferSizeHistoryMin = 0; + + std::queue<size_t> InflightRequestsHistory; + size_t InflightRequestsHistoryMin = 0; + }; + void Handle(TEvHttpBase::TEvSendResult::TPtr& ev) { - const auto* res = ev->Get(); - const TString& error = res->HttpIncomingResponse->Get()->GetError(); - - if (!error.empty()) { - TStringBuilder errorBuilder; - errorBuilder << "Error while sending request to monitoring api: " << error; - const auto& response = res->HttpIncomingResponse->Get()->Response; - if (response) { - errorBuilder << " " << response->GetObfuscatedData(); - } - - TIssues issues { TIssue(errorBuilder) }; - SINK_LOG_W("Got error response from solomon " << issues.ToString()); - Callbacks->OnSinkError(OutputIndex, issues, res->IsTerminal); - return; - } - - HandleSuccessSolomonResponse(*res->HttpIncomingResponse->Get()); - - while (TryToSendNextBatch()) {} - } - + const auto* res = ev->Get(); + const TString& error = res->HttpIncomingResponse->Get()->GetError(); + + if (!error.empty()) { + TStringBuilder errorBuilder; + errorBuilder << "Error while sending request to monitoring api: " << error; + const auto& response = res->HttpIncomingResponse->Get()->Response; + if (response) { + errorBuilder << " " << response->GetObfuscatedData(); + } + + TIssues issues { TIssue(errorBuilder) }; + SINK_LOG_W("Got error response from solomon " << issues.ToString()); + Callbacks->OnSinkError(OutputIndex, issues, res->IsTerminal); + return; + } + + HandleSuccessSolomonResponse(*res->HttpIncomingResponse->Get()); + + while (TryToSendNextBatch()) {} + } + // IActor & IDqSinkActor void PassAway() override { // Is called from Compute Actor - for (const auto& [_, metricsInflight] : InflightBuffer) { - Send(metricsInflight.HttpSenderId, new TEvents::TEvPoison()); - } - - if (HttpProxyId) { - Send(HttpProxyId, new TEvents::TEvPoison()); - } - + for (const auto& [_, metricsInflight] : InflightBuffer) { + Send(metricsInflight.HttpSenderId, new TEvents::TEvPoison()); + } + + if (HttpProxyId) { + Send(HttpProxyId, new TEvents::TEvPoison()); + } + TActor<TDqSolomonWriteActor>::PassAway(); - } - -private: + } + +private: NDqProto::TSinkState BuildState() { return {}; } - - TString GetUrl() const { - TStringBuilder builder; + + TString GetUrl() const { + TStringBuilder builder; builder << (WriteParams.Shard.GetUseSsl() ? "https://" : "http://"); - builder << WriteParams.Shard.GetEndpoint(); - - switch (WriteParams.Shard.GetClusterType()) { - case NSo::NProto::ESolomonClusterType::CT_SOLOMON: { - builder << "/api/v2/push"; - builder << "?project=" << WriteParams.Shard.GetProject(); - builder << "&cluster=" << WriteParams.Shard.GetCluster(); - builder << "&service=" << WriteParams.Shard.GetService(); - break; - } - case NSo::NProto::ESolomonClusterType::CT_MONITORING: { - builder << "/monitoring/v2/data/write"; - builder << "?folderId=" << WriteParams.Shard.GetCluster(); - builder << "&service=" << WriteParams.Shard.GetService(); - break; - } - default: - Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType())); - } - - return builder; - } - - void PushMetricsToBuffer(ui64& metricsCount) { - try { - auto data = UserMetricsEncoder.Encode(); - SINK_LOG_D("Push " << data.size() << " bytes of data to buffer"); - - FreeSpace -= data.size(); - SendingBuffer.emplace(TMetricsToSend { std::move(data), metricsCount }); - } catch (const yexception& e) { - TIssues issues { TIssue(TStringBuilder() << "Error while encoding solomon metrics: " << e.what()) }; - Callbacks->OnSinkError(OutputIndex, issues, true); - } - - metricsCount = 0; - } - - NHttp::THttpOutgoingRequestPtr BuildSolomonRequest(const TString& data) { - NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestPost(Url); - FillAuth(httpRequest); - httpRequest->Set<&NHttp::THttpRequest::ContentType>("application/json"); - httpRequest->Set<&NHttp::THttpRequest::Body>(data); - return httpRequest; - } - - void FillAuth(NHttp::THttpOutgoingRequestPtr& httpRequest) { - const TString authorizationHeader = "Authorization"; + builder << WriteParams.Shard.GetEndpoint(); + + switch (WriteParams.Shard.GetClusterType()) { + case NSo::NProto::ESolomonClusterType::CT_SOLOMON: { + builder << "/api/v2/push"; + builder << "?project=" << WriteParams.Shard.GetProject(); + builder << "&cluster=" << WriteParams.Shard.GetCluster(); + builder << "&service=" << WriteParams.Shard.GetService(); + break; + } + case NSo::NProto::ESolomonClusterType::CT_MONITORING: { + builder << "/monitoring/v2/data/write"; + builder << "?folderId=" << WriteParams.Shard.GetCluster(); + builder << "&service=" << WriteParams.Shard.GetService(); + break; + } + default: + Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType())); + } + + return builder; + } + + void PushMetricsToBuffer(ui64& metricsCount) { + try { + auto data = UserMetricsEncoder.Encode(); + SINK_LOG_D("Push " << data.size() << " bytes of data to buffer"); + + FreeSpace -= data.size(); + SendingBuffer.emplace(TMetricsToSend { std::move(data), metricsCount }); + } catch (const yexception& e) { + TIssues issues { TIssue(TStringBuilder() << "Error while encoding solomon metrics: " << e.what()) }; + Callbacks->OnSinkError(OutputIndex, issues, true); + } + + metricsCount = 0; + } + + NHttp::THttpOutgoingRequestPtr BuildSolomonRequest(const TString& data) { + NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestPost(Url); + FillAuth(httpRequest); + httpRequest->Set<&NHttp::THttpRequest::ContentType>("application/json"); + httpRequest->Set<&NHttp::THttpRequest::Body>(data); + return httpRequest; + } + + void FillAuth(NHttp::THttpOutgoingRequestPtr& httpRequest) { + const TString authorizationHeader = "Authorization"; const TString authToken = CredentialsProvider->GetAuthInfo(); - - switch (WriteParams.Shard.GetClusterType()) { - case NSo::NProto::ESolomonClusterType::CT_SOLOMON: + + switch (WriteParams.Shard.GetClusterType()) { + case NSo::NProto::ESolomonClusterType::CT_SOLOMON: httpRequest->Set(authorizationHeader, "OAuth " + authToken); - break; - case NSo::NProto::ESolomonClusterType::CT_MONITORING: + break; + case NSo::NProto::ESolomonClusterType::CT_MONITORING: httpRequest->Set(authorizationHeader, "Bearer " + authToken); - break; - default: - Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType())); - } - } - - bool TryToSendNextBatch() { - Metrics.ReportSendingBufferSize(SendingBuffer.size()); - Metrics.ReportInflightRequestsCount(InflightBuffer.size()); - - if (CheckpointInProgress || SendingBuffer.empty() || InflightBuffer.size() >= MaxRequestsInflight) { - TStringBuilder skipReason; - skipReason << "Skip sending to solomon. Reason: "; - if (CheckpointInProgress) { - skipReason << "CheckpointInProgress "; - } - if (SendingBuffer.empty()) { - skipReason << "Empty buffer "; - } - if (InflightBuffer.size() >= MaxRequestsInflight) { - skipReason << "MaxRequestsInflight "; - } - SINK_LOG_D(skipReason); - return false; - } - - auto variant = SendingBuffer.front(); - SendingBuffer.pop(); - - if (std::holds_alternative<TMetricsToSend>(variant)) { - if (Y_UNLIKELY(!HttpProxyId)) { - HttpProxyId = Register(NHttp::CreateHttpProxy(*NMonitoring::TMetricRegistry::Instance())); - } - - const auto metricsToSend = std::get<TMetricsToSend>(variant); - const NHttp::THttpOutgoingRequestPtr httpRequest = BuildSolomonRequest(metricsToSend.Data); - - const TActorId httpSenderId = Register(CreateHttpSenderActor(SelfId(), HttpProxyId)); - Send(httpSenderId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest)); - SINK_LOG_D("Sent " << metricsToSend.MetricsCount << " metrics with size of " << metricsToSend.Data.size() << " bytes to solomon"); - - *Metrics.SentMetrics += metricsToSend.MetricsCount; - InflightBuffer.emplace(httpRequest, TMetricsInflight { httpSenderId, metricsToSend.MetricsCount }); - return true; - } - - if (std::holds_alternative<NDqProto::TCheckpoint>(variant)) { - CheckpointInProgress = std::get<NDqProto::TCheckpoint>(std::move(variant)); - if (InflightBuffer.empty()) { - DoCheckpoint(); - return true; - } - return false; - } - - Y_ENSURE(false, "Bad type"); - } - - void HandleSuccessSolomonResponse(const NHttp::TEvHttpProxy::TEvHttpIncomingResponse& response) { - SINK_LOG_D("Solomon response: " << response.Response->GetObfuscatedData()); - NJson::TJsonParser parser; - switch (WriteParams.Shard.GetClusterType()) { - case NSo::NProto::ESolomonClusterType::CT_SOLOMON: - parser.AddField("sensorsProcessed", true); - break; - case NSo::NProto::ESolomonClusterType::CT_MONITORING: - parser.AddField("writtenMetricsCount", true); - break; - default: - Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType())); - } - parser.AddField("errorMessage", false); - - TVector<TString> res; - if (!parser.Parse(TString(response.Response->Body), &res)) { - TIssues issues { TIssue(TStringBuilder() << "Invalid monitoring response: " << response.Response->GetObfuscatedData()) }; - Callbacks->OnSinkError(OutputIndex, issues, true); - return; - } - Y_VERIFY(res.size() == 2); - - auto ptr = InflightBuffer.find(response.Request); - Y_VERIFY(ptr != InflightBuffer.end()); - - const ui64 writtenMetricsCount = std::stoul(res[0]); - *Metrics.СonfirmedMetrics += writtenMetricsCount; - if (writtenMetricsCount != ptr->second.MetricsCount) { - // TODO: YQ-340 - // TIssues issues { TIssue(TStringBuilder() << ToString(ptr->second.MetricsCount - writtenMetricsCount) << " metrics were not written: " << res[1]) }; - // Callbacks->OnSinkError(OutputIndex, issues, true); - // return; - SINK_LOG_W("Some metrics were not written. MetricsCount=" << ptr->second.MetricsCount << " writtenMetricsCount=" << writtenMetricsCount << " Solomon response: " << response.Response->GetObfuscatedData()); - } - - FreeSpace += ptr->first->Body.Size(); - if (ShouldNotifyNewFreeSpace) { - Callbacks->ResumeExecution(); - ShouldNotifyNewFreeSpace = false; - } - InflightBuffer.erase(ptr); - - if (CheckpointInProgress && InflightBuffer.empty()) { - DoCheckpoint(); - } - } - - void DoCheckpoint() { + break; + default: + Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType())); + } + } + + bool TryToSendNextBatch() { + Metrics.ReportSendingBufferSize(SendingBuffer.size()); + Metrics.ReportInflightRequestsCount(InflightBuffer.size()); + + if (CheckpointInProgress || SendingBuffer.empty() || InflightBuffer.size() >= MaxRequestsInflight) { + TStringBuilder skipReason; + skipReason << "Skip sending to solomon. Reason: "; + if (CheckpointInProgress) { + skipReason << "CheckpointInProgress "; + } + if (SendingBuffer.empty()) { + skipReason << "Empty buffer "; + } + if (InflightBuffer.size() >= MaxRequestsInflight) { + skipReason << "MaxRequestsInflight "; + } + SINK_LOG_D(skipReason); + return false; + } + + auto variant = SendingBuffer.front(); + SendingBuffer.pop(); + + if (std::holds_alternative<TMetricsToSend>(variant)) { + if (Y_UNLIKELY(!HttpProxyId)) { + HttpProxyId = Register(NHttp::CreateHttpProxy(*NMonitoring::TMetricRegistry::Instance())); + } + + const auto metricsToSend = std::get<TMetricsToSend>(variant); + const NHttp::THttpOutgoingRequestPtr httpRequest = BuildSolomonRequest(metricsToSend.Data); + + const TActorId httpSenderId = Register(CreateHttpSenderActor(SelfId(), HttpProxyId)); + Send(httpSenderId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest)); + SINK_LOG_D("Sent " << metricsToSend.MetricsCount << " metrics with size of " << metricsToSend.Data.size() << " bytes to solomon"); + + *Metrics.SentMetrics += metricsToSend.MetricsCount; + InflightBuffer.emplace(httpRequest, TMetricsInflight { httpSenderId, metricsToSend.MetricsCount }); + return true; + } + + if (std::holds_alternative<NDqProto::TCheckpoint>(variant)) { + CheckpointInProgress = std::get<NDqProto::TCheckpoint>(std::move(variant)); + if (InflightBuffer.empty()) { + DoCheckpoint(); + return true; + } + return false; + } + + Y_ENSURE(false, "Bad type"); + } + + void HandleSuccessSolomonResponse(const NHttp::TEvHttpProxy::TEvHttpIncomingResponse& response) { + SINK_LOG_D("Solomon response: " << response.Response->GetObfuscatedData()); + NJson::TJsonParser parser; + switch (WriteParams.Shard.GetClusterType()) { + case NSo::NProto::ESolomonClusterType::CT_SOLOMON: + parser.AddField("sensorsProcessed", true); + break; + case NSo::NProto::ESolomonClusterType::CT_MONITORING: + parser.AddField("writtenMetricsCount", true); + break; + default: + Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType())); + } + parser.AddField("errorMessage", false); + + TVector<TString> res; + if (!parser.Parse(TString(response.Response->Body), &res)) { + TIssues issues { TIssue(TStringBuilder() << "Invalid monitoring response: " << response.Response->GetObfuscatedData()) }; + Callbacks->OnSinkError(OutputIndex, issues, true); + return; + } + Y_VERIFY(res.size() == 2); + + auto ptr = InflightBuffer.find(response.Request); + Y_VERIFY(ptr != InflightBuffer.end()); + + const ui64 writtenMetricsCount = std::stoul(res[0]); + *Metrics.СonfirmedMetrics += writtenMetricsCount; + if (writtenMetricsCount != ptr->second.MetricsCount) { + // TODO: YQ-340 + // TIssues issues { TIssue(TStringBuilder() << ToString(ptr->second.MetricsCount - writtenMetricsCount) << " metrics were not written: " << res[1]) }; + // Callbacks->OnSinkError(OutputIndex, issues, true); + // return; + SINK_LOG_W("Some metrics were not written. MetricsCount=" << ptr->second.MetricsCount << " writtenMetricsCount=" << writtenMetricsCount << " Solomon response: " << response.Response->GetObfuscatedData()); + } + + FreeSpace += ptr->first->Body.Size(); + if (ShouldNotifyNewFreeSpace) { + Callbacks->ResumeExecution(); + ShouldNotifyNewFreeSpace = false; + } + InflightBuffer.erase(ptr); + + if (CheckpointInProgress && InflightBuffer.empty()) { + DoCheckpoint(); + } + } + + void DoCheckpoint() { Callbacks->OnSinkStateSaved(BuildState(), OutputIndex, *CheckpointInProgress); - CheckpointInProgress = std::nullopt; - } - -private: - const ui64 OutputIndex; - const TDqSolomonWriteParams WriteParams; - const TString Url; - NYql::NDq::IDqSinkActor::ICallbacks* const Callbacks; - TDqSolomonWriteActorMetrics Metrics; - i64 FreeSpace = 0; - TActorId HttpProxyId; - - TString SourceId; - bool ShouldNotifyNewFreeSpace = false; - std::optional<NDqProto::TCheckpoint> CheckpointInProgress = std::nullopt; - std::queue<std::variant<TMetricsToSend, NDqProto::TCheckpoint>> SendingBuffer; - THashMap<NHttp::THttpOutgoingRequestPtr, TMetricsInflight> InflightBuffer; - - TMetricsEncoder UserMetricsEncoder; + CheckpointInProgress = std::nullopt; + } + +private: + const ui64 OutputIndex; + const TDqSolomonWriteParams WriteParams; + const TString Url; + NYql::NDq::IDqSinkActor::ICallbacks* const Callbacks; + TDqSolomonWriteActorMetrics Metrics; + i64 FreeSpace = 0; + TActorId HttpProxyId; + + TString SourceId; + bool ShouldNotifyNewFreeSpace = false; + std::optional<NDqProto::TCheckpoint> CheckpointInProgress = std::nullopt; + std::queue<std::variant<TMetricsToSend, NDqProto::TCheckpoint>> SendingBuffer; + THashMap<NHttp::THttpOutgoingRequestPtr, TMetricsInflight> InflightBuffer; + + TMetricsEncoder UserMetricsEncoder; std::shared_ptr<NYdb::ICredentialsProvider> CredentialsProvider; -}; - -std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor( - NYql::NSo::NProto::TDqSolomonShard&& settings, - ui64 outputIndex, - const THashMap<TString, TString>& secureParams, - NYql::NDq::IDqSinkActor::ICallbacks* callbacks, +}; + +std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor( + NYql::NSo::NProto::TDqSolomonShard&& settings, + ui64 outputIndex, + const THashMap<TString, TString>& secureParams, + NYql::NDq::IDqSinkActor::ICallbacks* callbacks, const NMonitoring::TDynamicCounterPtr& counters, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, - i64 freeSpace) -{ + i64 freeSpace) +{ const TString& tokenName = settings.GetToken().GetName(); const TString token = secureParams.Value(tokenName, TString()); - - TDqSolomonWriteParams params { - .Shard = std::move(settings), - }; - + + TDqSolomonWriteParams params { + .Shard = std::move(settings), + }; + auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token); auto credentialsProvider = credentialsProviderFactory->CreateProvider(); - TDqSolomonWriteActor* actor = new TDqSolomonWriteActor( - outputIndex, - std::move(params), - callbacks, - counters, + TDqSolomonWriteActor* actor = new TDqSolomonWriteActor( + outputIndex, + std::move(params), + callbacks, + counters, credentialsProvider, - freeSpace); - return {actor, actor}; -} - + freeSpace); + return {actor, actor}; +} + void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { - factory.Register<NSo::NProto::TDqSolomonShard>("SolomonSink", + factory.Register<NSo::NProto::TDqSolomonShard>("SolomonSink", [credentialsFactory]( - NYql::NSo::NProto::TDqSolomonShard&& settings, - IDqSinkActorFactory::TArguments&& args) - { - auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); - - return CreateDqSolomonWriteActor( - std::move(settings), - args.OutputIndex, - args.SecureParams, - args.Callback, + NYql::NSo::NProto::TDqSolomonShard&& settings, + IDqSinkActorFactory::TArguments&& args) + { + auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + + return CreateDqSolomonWriteActor( + std::move(settings), + args.OutputIndex, + args.SecureParams, + args.Callback, counters, credentialsFactory); - }); -} - -} // namespace NYql::NDq + }); +} + +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h index 46810fca144..56c34cbb25f 100644 --- a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h +++ b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h @@ -1,32 +1,32 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/utils/actors/http_sender.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/providers/solomon/proto/dq_solomon_shard.pb.h> - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/http/http_proxy.h> - -#include <util/generic/size_literals.h> -#include <util/system/types.h> - -namespace NYql::NDq { - -constexpr i64 DqSolomonDefaultFreeSpace = 16_MB; - -std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor( - NYql::NSo::NProto::TDqSolomonShard&& settings, - ui64 outputIndex, - const THashMap<TString, TString>& secureParams, - NYql::NDq::IDqSinkActor::ICallbacks* callbacks, + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/http/http_proxy.h> + +#include <util/generic/size_literals.h> +#include <util/system/types.h> + +namespace NYql::NDq { + +constexpr i64 DqSolomonDefaultFreeSpace = 16_MB; + +std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor( + NYql::NSo::NProto::TDqSolomonShard&& settings, + ui64 outputIndex, + const THashMap<TString, TString>& secureParams, + NYql::NDq::IDqSinkActor::ICallbacks* callbacks, const NMonitoring::TDynamicCounterPtr& counters, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, - i64 freeSpace = DqSolomonDefaultFreeSpace); - + i64 freeSpace = DqSolomonDefaultFreeSpace); + void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); - -} // namespace NYql::NDq + +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp b/ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp index f3e7e571771..80637642fb3 100644 --- a/ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp +++ b/ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp @@ -1,149 +1,149 @@ -#include "metrics_encoder.h" - +#include "metrics_encoder.h" + #include <ydb/library/yql/public/udf/udf_data_type.h> - -#include <util/datetime/base.h> - -using namespace NYql; - -namespace { - -void BeginMetric( - const NMonitoring::IMetricEncoderPtr& encoder, - const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme) -{ - switch (scheme.GetDataTypeId()) { - case NUdf::TDataType<i8>::Id: - case NUdf::TDataType<ui8>::Id: - case NUdf::TDataType<i16>::Id: - case NUdf::TDataType<ui16>::Id: - case NUdf::TDataType<i32>::Id: - case NUdf::TDataType<ui32>::Id: - case NUdf::TDataType<i64>::Id: - case NUdf::TDataType<ui64>::Id: - encoder->OnMetricBegin(NMonitoring::EMetricType::IGAUGE); - break; - case NUdf::TDataType<float>::Id: - case NUdf::TDataType<double>::Id: - encoder->OnMetricBegin(NMonitoring::EMetricType::GAUGE); - break; - default: - Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId()); - } -} - -TInstant ParseTimestamp( - const NUdf::TUnboxedValue& unboxedValue, - const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme) -{ - const auto& timestampValue = unboxedValue.GetElement(scheme.GetIndex()); - switch (scheme.GetDataTypeId()) { - case NUdf::TDataType<NUdf::TDate>::Id: - case NUdf::TDataType<NUdf::TTzDate>::Id: - case NUdf::TDataType<NUdf::TDatetime>::Id: - case NUdf::TDataType<NUdf::TTzDatetime>::Id: - return TInstant::Seconds(timestampValue.Get<ui32>()); - case NUdf::TDataType<NUdf::TTimestamp>::Id: - case NUdf::TDataType<NUdf::TTzTimestamp>::Id: - return TInstant::MicroSeconds(timestampValue.Get<ui64>()); - default: - Y_ENSURE(false, "Bad type for timestamp " << scheme.GetDataTypeId()); - } -} - -void EncodeSensorValue( - const NMonitoring::IMetricEncoderPtr& encoder, - const TInstant& timestamp, - const NUdf::TUnboxedValue& unboxedValue, - const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme) -{ - switch (scheme.GetDataTypeId()) { - case NUdf::TDataType<i8>::Id: - encoder->OnInt64(timestamp, unboxedValue.Get<i8>()); - break; - case NUdf::TDataType<ui8>::Id: - encoder->OnUint64(timestamp, unboxedValue.Get<ui8>()); - break; - case NUdf::TDataType<i16>::Id: - encoder->OnInt64(timestamp, unboxedValue.Get<i16>()); - break; - case NUdf::TDataType<ui16>::Id: - encoder->OnUint64(timestamp, unboxedValue.Get<ui16>()); - break; - case NUdf::TDataType<i32>::Id: - encoder->OnInt64(timestamp, unboxedValue.Get<i32>()); - break; - case NUdf::TDataType<ui32>::Id: - encoder->OnUint64(timestamp, unboxedValue.Get<ui32>()); - break; - case NUdf::TDataType<i64>::Id: - encoder->OnInt64(timestamp, unboxedValue.Get<i64>()); - break; - case NUdf::TDataType<ui64>::Id: - encoder->OnUint64(timestamp, unboxedValue.Get<ui64>()); - break; - case NUdf::TDataType<float>::Id: - encoder->OnDouble(timestamp, unboxedValue.Get<float>()); - break; - case NUdf::TDataType<double>::Id: - encoder->OnDouble(timestamp, unboxedValue.Get<double>()); - break; - default: - Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId()); - } -} -} - -namespace NYql::NDq { - -TMetricsEncoder::TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool useCloudFormat) - : Scheme(scheme) - , UseCloudFormat(useCloudFormat) -{ - BeginNew(); -} - -void TMetricsEncoder::BeginNew() { - Data.clear(); - DataOut.emplace(Data); - SolomonEncoder = UseCloudFormat - ? NMonitoring::BufferedEncoderCloudJson(&DataOut.value(), 0, "name") - : NMonitoring::BufferedEncoderJson(&DataOut.value(), 0); - - SolomonEncoder->OnStreamBegin(); -} - -ui64 TMetricsEncoder::Append(const NUdf::TUnboxedValue& value) -{ - TInstant timestamp = ParseTimestamp(value, Scheme.GetTimestamp()); - - for (const auto& sensor : Scheme.GetSensors()) { - BeginMetric(SolomonEncoder, sensor); - - SolomonEncoder->OnLabelsBegin(); - SolomonEncoder->OnLabel("name", sensor.GetKey()); - for (const auto& label : Scheme.GetLabels()) { - const NUdf::TUnboxedValue& labelValue = value.GetElement(label.GetIndex()); - SolomonEncoder->OnLabel(label.GetKey(), TString(labelValue.AsStringRef())); - } - SolomonEncoder->OnLabelsEnd(); - - const auto& sensorValue = value.GetElement(sensor.GetIndex()); - EncodeSensorValue(SolomonEncoder, timestamp, sensorValue, sensor); - - SolomonEncoder->OnMetricEnd(); - } - - return Scheme.GetSensors().size(); -} - -TString TMetricsEncoder::Encode() { - SolomonEncoder->OnStreamEnd(); - SolomonEncoder->Close(); - - TString res = Data; - BeginNew(); - return res; -} - -} + +#include <util/datetime/base.h> + +using namespace NYql; + +namespace { + +void BeginMetric( + const NMonitoring::IMetricEncoderPtr& encoder, + const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme) +{ + switch (scheme.GetDataTypeId()) { + case NUdf::TDataType<i8>::Id: + case NUdf::TDataType<ui8>::Id: + case NUdf::TDataType<i16>::Id: + case NUdf::TDataType<ui16>::Id: + case NUdf::TDataType<i32>::Id: + case NUdf::TDataType<ui32>::Id: + case NUdf::TDataType<i64>::Id: + case NUdf::TDataType<ui64>::Id: + encoder->OnMetricBegin(NMonitoring::EMetricType::IGAUGE); + break; + case NUdf::TDataType<float>::Id: + case NUdf::TDataType<double>::Id: + encoder->OnMetricBegin(NMonitoring::EMetricType::GAUGE); + break; + default: + Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId()); + } +} + +TInstant ParseTimestamp( + const NUdf::TUnboxedValue& unboxedValue, + const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme) +{ + const auto& timestampValue = unboxedValue.GetElement(scheme.GetIndex()); + switch (scheme.GetDataTypeId()) { + case NUdf::TDataType<NUdf::TDate>::Id: + case NUdf::TDataType<NUdf::TTzDate>::Id: + case NUdf::TDataType<NUdf::TDatetime>::Id: + case NUdf::TDataType<NUdf::TTzDatetime>::Id: + return TInstant::Seconds(timestampValue.Get<ui32>()); + case NUdf::TDataType<NUdf::TTimestamp>::Id: + case NUdf::TDataType<NUdf::TTzTimestamp>::Id: + return TInstant::MicroSeconds(timestampValue.Get<ui64>()); + default: + Y_ENSURE(false, "Bad type for timestamp " << scheme.GetDataTypeId()); + } +} + +void EncodeSensorValue( + const NMonitoring::IMetricEncoderPtr& encoder, + const TInstant& timestamp, + const NUdf::TUnboxedValue& unboxedValue, + const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme) +{ + switch (scheme.GetDataTypeId()) { + case NUdf::TDataType<i8>::Id: + encoder->OnInt64(timestamp, unboxedValue.Get<i8>()); + break; + case NUdf::TDataType<ui8>::Id: + encoder->OnUint64(timestamp, unboxedValue.Get<ui8>()); + break; + case NUdf::TDataType<i16>::Id: + encoder->OnInt64(timestamp, unboxedValue.Get<i16>()); + break; + case NUdf::TDataType<ui16>::Id: + encoder->OnUint64(timestamp, unboxedValue.Get<ui16>()); + break; + case NUdf::TDataType<i32>::Id: + encoder->OnInt64(timestamp, unboxedValue.Get<i32>()); + break; + case NUdf::TDataType<ui32>::Id: + encoder->OnUint64(timestamp, unboxedValue.Get<ui32>()); + break; + case NUdf::TDataType<i64>::Id: + encoder->OnInt64(timestamp, unboxedValue.Get<i64>()); + break; + case NUdf::TDataType<ui64>::Id: + encoder->OnUint64(timestamp, unboxedValue.Get<ui64>()); + break; + case NUdf::TDataType<float>::Id: + encoder->OnDouble(timestamp, unboxedValue.Get<float>()); + break; + case NUdf::TDataType<double>::Id: + encoder->OnDouble(timestamp, unboxedValue.Get<double>()); + break; + default: + Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId()); + } +} +} + +namespace NYql::NDq { + +TMetricsEncoder::TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool useCloudFormat) + : Scheme(scheme) + , UseCloudFormat(useCloudFormat) +{ + BeginNew(); +} + +void TMetricsEncoder::BeginNew() { + Data.clear(); + DataOut.emplace(Data); + SolomonEncoder = UseCloudFormat + ? NMonitoring::BufferedEncoderCloudJson(&DataOut.value(), 0, "name") + : NMonitoring::BufferedEncoderJson(&DataOut.value(), 0); + + SolomonEncoder->OnStreamBegin(); +} + +ui64 TMetricsEncoder::Append(const NUdf::TUnboxedValue& value) +{ + TInstant timestamp = ParseTimestamp(value, Scheme.GetTimestamp()); + + for (const auto& sensor : Scheme.GetSensors()) { + BeginMetric(SolomonEncoder, sensor); + + SolomonEncoder->OnLabelsBegin(); + SolomonEncoder->OnLabel("name", sensor.GetKey()); + for (const auto& label : Scheme.GetLabels()) { + const NUdf::TUnboxedValue& labelValue = value.GetElement(label.GetIndex()); + SolomonEncoder->OnLabel(label.GetKey(), TString(labelValue.AsStringRef())); + } + SolomonEncoder->OnLabelsEnd(); + + const auto& sensorValue = value.GetElement(sensor.GetIndex()); + EncodeSensorValue(SolomonEncoder, timestamp, sensorValue, sensor); + + SolomonEncoder->OnMetricEnd(); + } + + return Scheme.GetSensors().size(); +} + +TString TMetricsEncoder::Encode() { + SolomonEncoder->OnStreamEnd(); + SolomonEncoder->Close(); + + TString res = Data; + BeginNew(); + return res; +} + +} diff --git a/ydb/library/yql/providers/solomon/async_io/metrics_encoder.h b/ydb/library/yql/providers/solomon/async_io/metrics_encoder.h index 2826a3ca4ac..b759e288e46 100644 --- a/ydb/library/yql/providers/solomon/async_io/metrics_encoder.h +++ b/ydb/library/yql/providers/solomon/async_io/metrics_encoder.h @@ -1,31 +1,31 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/utils/actors/http_sender.h> - + #include <ydb/library/yql/public/udf/udf_value.h> - -#include <library/cpp/monlib/encode/json/json.h> - -#include <util/stream/str.h> - - -namespace NYql::NDq { - -class TMetricsEncoder { -public: - explicit TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool cloudFormat); - - void BeginNew(); - ui64 Append(const NUdf::TUnboxedValue& value); - TString Encode(); - -private: - const NSo::NProto::TDqSolomonShardScheme& Scheme; - - TString Data; - std::optional<TStringOutput> DataOut; - NMonitoring::IMetricEncoderPtr SolomonEncoder; - bool UseCloudFormat; -}; - -} + +#include <library/cpp/monlib/encode/json/json.h> + +#include <util/stream/str.h> + + +namespace NYql::NDq { + +class TMetricsEncoder { +public: + explicit TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool cloudFormat); + + void BeginNew(); + ui64 Append(const NUdf::TUnboxedValue& value); + TString Encode(); + +private: + const NSo::NProto::TDqSolomonShardScheme& Scheme; + + TString Data; + std::optional<TStringOutput> DataOut; + NMonitoring::IMetricEncoderPtr SolomonEncoder; + bool UseCloudFormat; +}; + +} diff --git a/ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp b/ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp index d9de9f17ed6..3fca3a2bdc5 100644 --- a/ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp +++ b/ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp @@ -1,184 +1,184 @@ -#include "ut_helpers.h" - +#include "ut_helpers.h" + #include <ydb/library/yql/minikql/mkql_string_util.h> #include <ydb/library/yql/utils/yql_panic.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <thread> - -namespace NYql::NDq { - -using namespace NKikimr::NMiniKQL; - -constexpr TDuration WaitTimeout = TDuration::Seconds(10); - -namespace { - void TestWriteBigBatch(bool isCloud) { - const int batchSize = 7500; - CleanupSolomon("cloudId1", "folderId1", "custom", isCloud); - - TFakeCASetup setup; - InitSink(setup, BuildSolomonShardSettings(isCloud)); - - auto issue = setup.SinkPromises.Issue.GetFuture(); - setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ - TUnboxedValueVector res; - res.reserve(batchSize); - - for (int i = 0; i < batchSize; i++) { - res.emplace_back(CreateStruct(holderFactory, { - NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)), - NKikimr::NMiniKQL::MakeString(std::to_string(i)), - NUdf::TUnboxedValuePod(678) - })); - } - - return res; - }); - UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString()); - - const auto metrics = GetSolomonMetrics("folderId1", "custom"); - UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize); - } -} - -Y_UNIT_TEST_SUITE(TDqSolomonWriteActorTest) { - Y_UNIT_TEST(TestWriteFormat) { - CleanupSolomon("cloudId1", "folderId1", "custom", true); - - TFakeCASetup setup; - InitSink(setup, BuildSolomonShardSettings(true)); - - auto issue = setup.SinkPromises.Issue.GetFuture(); - setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ - NUdf::TUnboxedValue val1 = CreateStruct(holderFactory, { - NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(1624811684)), - NKikimr::NMiniKQL::MakeString("123"), - NUdf::TUnboxedValuePod(678) - }); - return TUnboxedValueVector { std::move(val1) }; - }); - UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString()); - - const auto metrics = GetSolomonMetrics("folderId1", "custom"); - const auto expected = R"([ - { - "labels": [ - [ - "label1", - "123" - ], - [ - "name", - "sensor1" - ] - ], - "ts": "1970-01-01T00:27:04.811684Z", - "value": 678 - } -])"; - UNIT_ASSERT_STRINGS_EQUAL(metrics, expected); - } - - Y_UNIT_TEST(TestWriteBigBatchMonitoring) { - TestWriteBigBatch(true); - } - - Y_UNIT_TEST(TestWriteBigBatchSolomon) { - //TestWriteBigBatch(false); - } - - Y_UNIT_TEST(TestWriteWithTimeseries) { - const int batchSize = 10; - CleanupSolomon("cloudId1", "folderId1", "custom", true); - - TFakeCASetup setup; - InitSink(setup, BuildSolomonShardSettings(true)); - - auto issue = setup.SinkPromises.Issue.GetFuture(); - setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ - TUnboxedValueVector res; - res.reserve(batchSize); - - for (int i = 0; i < batchSize; i++) { - res.emplace_back(CreateStruct(holderFactory, { - NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)), - NKikimr::NMiniKQL::MakeString("123"), - NUdf::TUnboxedValuePod(678) - })); - } - - return res; - }); - UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString()); - - const auto metrics = GetSolomonMetrics("folderId1", "custom"); - UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize); - } - - Y_UNIT_TEST(TestCheckpoints) { - const int batchSize = 2400; - - { - TFakeCASetup setup; - CleanupSolomon("cloudId1", "folderId1", "custom", true); - InitSink(setup, BuildSolomonShardSettings(true)); - - auto stateSaved = setup.SinkPromises.StateSaved.GetFuture(); - setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ - TUnboxedValueVector res; - res.reserve(batchSize); - - for (int i = 0; i < batchSize; i++) { - res.emplace_back(CreateStruct(holderFactory, { - NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)), - NKikimr::NMiniKQL::MakeString(std::to_string(i)), - NUdf::TUnboxedValuePod(678) - })); - } - - return res; - }, CreateCheckpoint(1)); - UNIT_ASSERT(stateSaved.Wait(WaitTimeout)); - - const auto metrics = GetSolomonMetrics("folderId1", "custom"); - UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize); - } - } - - Y_UNIT_TEST(TestShouldReturnAfterCheckpoint) { - { - TFakeCASetup setup; - CleanupSolomon("cloudId1", "folderId1", "custom", true); - InitSink(setup, BuildSolomonShardSettings(true)); - - auto stateSaved = setup.SinkPromises.StateSaved.GetFuture(); - setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ - return TUnboxedValueVector { - CreateStruct(holderFactory, { - NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(200000)), - NKikimr::NMiniKQL::MakeString("abc"), - NUdf::TUnboxedValuePod(678) - })}; - }, CreateCheckpoint(1)); - UNIT_ASSERT(stateSaved.Wait(WaitTimeout)); - - auto issue = setup.SinkPromises.Issue.GetFuture(); - setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ - return TUnboxedValueVector { - CreateStruct(holderFactory, { - NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(200001)), - NKikimr::NMiniKQL::MakeString("cba"), - NUdf::TUnboxedValuePod(678) - })}; - }); - UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString()); - - const auto metrics = GetSolomonMetrics("folderId1", "custom"); - UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), 2); - } - } -} - -} // namespace NKikimr::NMiniKQL + +#include <library/cpp/testing/unittest/registar.h> + +#include <thread> + +namespace NYql::NDq { + +using namespace NKikimr::NMiniKQL; + +constexpr TDuration WaitTimeout = TDuration::Seconds(10); + +namespace { + void TestWriteBigBatch(bool isCloud) { + const int batchSize = 7500; + CleanupSolomon("cloudId1", "folderId1", "custom", isCloud); + + TFakeCASetup setup; + InitSink(setup, BuildSolomonShardSettings(isCloud)); + + auto issue = setup.SinkPromises.Issue.GetFuture(); + setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ + TUnboxedValueVector res; + res.reserve(batchSize); + + for (int i = 0; i < batchSize; i++) { + res.emplace_back(CreateStruct(holderFactory, { + NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)), + NKikimr::NMiniKQL::MakeString(std::to_string(i)), + NUdf::TUnboxedValuePod(678) + })); + } + + return res; + }); + UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString()); + + const auto metrics = GetSolomonMetrics("folderId1", "custom"); + UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize); + } +} + +Y_UNIT_TEST_SUITE(TDqSolomonWriteActorTest) { + Y_UNIT_TEST(TestWriteFormat) { + CleanupSolomon("cloudId1", "folderId1", "custom", true); + + TFakeCASetup setup; + InitSink(setup, BuildSolomonShardSettings(true)); + + auto issue = setup.SinkPromises.Issue.GetFuture(); + setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ + NUdf::TUnboxedValue val1 = CreateStruct(holderFactory, { + NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(1624811684)), + NKikimr::NMiniKQL::MakeString("123"), + NUdf::TUnboxedValuePod(678) + }); + return TUnboxedValueVector { std::move(val1) }; + }); + UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString()); + + const auto metrics = GetSolomonMetrics("folderId1", "custom"); + const auto expected = R"([ + { + "labels": [ + [ + "label1", + "123" + ], + [ + "name", + "sensor1" + ] + ], + "ts": "1970-01-01T00:27:04.811684Z", + "value": 678 + } +])"; + UNIT_ASSERT_STRINGS_EQUAL(metrics, expected); + } + + Y_UNIT_TEST(TestWriteBigBatchMonitoring) { + TestWriteBigBatch(true); + } + + Y_UNIT_TEST(TestWriteBigBatchSolomon) { + //TestWriteBigBatch(false); + } + + Y_UNIT_TEST(TestWriteWithTimeseries) { + const int batchSize = 10; + CleanupSolomon("cloudId1", "folderId1", "custom", true); + + TFakeCASetup setup; + InitSink(setup, BuildSolomonShardSettings(true)); + + auto issue = setup.SinkPromises.Issue.GetFuture(); + setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ + TUnboxedValueVector res; + res.reserve(batchSize); + + for (int i = 0; i < batchSize; i++) { + res.emplace_back(CreateStruct(holderFactory, { + NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)), + NKikimr::NMiniKQL::MakeString("123"), + NUdf::TUnboxedValuePod(678) + })); + } + + return res; + }); + UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString()); + + const auto metrics = GetSolomonMetrics("folderId1", "custom"); + UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize); + } + + Y_UNIT_TEST(TestCheckpoints) { + const int batchSize = 2400; + + { + TFakeCASetup setup; + CleanupSolomon("cloudId1", "folderId1", "custom", true); + InitSink(setup, BuildSolomonShardSettings(true)); + + auto stateSaved = setup.SinkPromises.StateSaved.GetFuture(); + setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ + TUnboxedValueVector res; + res.reserve(batchSize); + + for (int i = 0; i < batchSize; i++) { + res.emplace_back(CreateStruct(holderFactory, { + NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)), + NKikimr::NMiniKQL::MakeString(std::to_string(i)), + NUdf::TUnboxedValuePod(678) + })); + } + + return res; + }, CreateCheckpoint(1)); + UNIT_ASSERT(stateSaved.Wait(WaitTimeout)); + + const auto metrics = GetSolomonMetrics("folderId1", "custom"); + UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize); + } + } + + Y_UNIT_TEST(TestShouldReturnAfterCheckpoint) { + { + TFakeCASetup setup; + CleanupSolomon("cloudId1", "folderId1", "custom", true); + InitSink(setup, BuildSolomonShardSettings(true)); + + auto stateSaved = setup.SinkPromises.StateSaved.GetFuture(); + setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ + return TUnboxedValueVector { + CreateStruct(holderFactory, { + NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(200000)), + NKikimr::NMiniKQL::MakeString("abc"), + NUdf::TUnboxedValuePod(678) + })}; + }, CreateCheckpoint(1)); + UNIT_ASSERT(stateSaved.Wait(WaitTimeout)); + + auto issue = setup.SinkPromises.Issue.GetFuture(); + setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){ + return TUnboxedValueVector { + CreateStruct(holderFactory, { + NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(200001)), + NKikimr::NMiniKQL::MakeString("cba"), + NUdf::TUnboxedValuePod(678) + })}; + }); + UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString()); + + const auto metrics = GetSolomonMetrics("folderId1", "custom"); + UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), 2); + } + } +} + +} // namespace NKikimr::NMiniKQL diff --git a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp index cf9cffdfa77..a97b2af7801 100644 --- a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp +++ b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp @@ -1,126 +1,126 @@ -#include "ut_helpers.h" - +#include "ut_helpers.h" + #include <ydb/core/testlib/basics/appdata.h> - -#include <library/cpp/http/simple/http_client.h> -#include <library/cpp/json/json_reader.h> -#include <library/cpp/retry/retry.h> - -#include <util/system/guard.h> - - -namespace NYql::NDq { - -using namespace NKikimr::NMiniKQL; - -namespace { - -void FillDqSolomonScheme(NSo::NProto::TDqSolomonShardScheme& scheme) { - scheme.MutableTimestamp()->SetKey("ts"); - scheme.MutableTimestamp()->SetIndex(0); - scheme.MutableTimestamp()->SetDataTypeId(NUdf::TDataType<NUdf::TTimestamp>::Id); - - NSo::NProto::TDqSolomonSchemeItem label; - label.SetKey("label1"); - label.SetIndex(1); - label.SetDataTypeId(NUdf::TDataType<ui32>::Id); - - NSo::NProto::TDqSolomonSchemeItem sensor; - sensor.SetKey("sensor1"); - sensor.SetIndex(2); - sensor.SetDataTypeId(NUdf::TDataType<ui32>::Id); - - scheme.MutableLabels()->Add(std::move(label)); - scheme.MutableSensors()->Add(std::move(sensor)); -} - -} - -void InitSink( - TFakeCASetup& caSetup, - NSo::NProto::TDqSolomonShard&& settings, - i64 freeSpace) -{ - auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); - const THashMap<TString, TString> secureParams; - - caSetup.Execute([&](TFakeActor& actor) { - auto [dqSink, dqSinkAsActor] = CreateDqSolomonWriteActor( - std::move(settings), - 0, - secureParams, - &actor.GetSinkCallbacks(), - counters, + +#include <library/cpp/http/simple/http_client.h> +#include <library/cpp/json/json_reader.h> +#include <library/cpp/retry/retry.h> + +#include <util/system/guard.h> + + +namespace NYql::NDq { + +using namespace NKikimr::NMiniKQL; + +namespace { + +void FillDqSolomonScheme(NSo::NProto::TDqSolomonShardScheme& scheme) { + scheme.MutableTimestamp()->SetKey("ts"); + scheme.MutableTimestamp()->SetIndex(0); + scheme.MutableTimestamp()->SetDataTypeId(NUdf::TDataType<NUdf::TTimestamp>::Id); + + NSo::NProto::TDqSolomonSchemeItem label; + label.SetKey("label1"); + label.SetIndex(1); + label.SetDataTypeId(NUdf::TDataType<ui32>::Id); + + NSo::NProto::TDqSolomonSchemeItem sensor; + sensor.SetKey("sensor1"); + sensor.SetIndex(2); + sensor.SetDataTypeId(NUdf::TDataType<ui32>::Id); + + scheme.MutableLabels()->Add(std::move(label)); + scheme.MutableSensors()->Add(std::move(sensor)); +} + +} + +void InitSink( + TFakeCASetup& caSetup, + NSo::NProto::TDqSolomonShard&& settings, + i64 freeSpace) +{ + auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + const THashMap<TString, TString> secureParams; + + caSetup.Execute([&](TFakeActor& actor) { + auto [dqSink, dqSinkAsActor] = CreateDqSolomonWriteActor( + std::move(settings), + 0, + secureParams, + &actor.GetSinkCallbacks(), + counters, nullptr, - freeSpace); - - actor.InitSink(dqSink, dqSinkAsActor); - }); -} - -void CleanupSolomon(TString cloudId, TString folderId, TString service, bool isCloud) { - const auto solomonPort = TString(getenv("SOLOMON_PORT")); - TSimpleHttpClient httpClient("localhost", std::stoi(solomonPort)); - TStringStream str; - TStringBuilder builder; - builder << "/cleanup"; - if (isCloud) { - builder << "?folderId=" << folderId << "&service=" << service; - } else { - builder << "?project=" << cloudId << "&cluster=" << folderId << "&service=" << service; - } - - DoWithRetry( - [&]{ httpClient.DoPost(builder, "", &str); }, - TRetryOptions(3, TDuration::Seconds(1)), - true); -} - -TString GetSolomonMetrics(TString folderId, TString service) { - const auto solomonPort = TString(getenv("SOLOMON_PORT")); - TSimpleHttpClient httpClient("localhost", std::stoi(solomonPort)); - TStringStream str; - httpClient.DoGet("/metrics?folderId=" + folderId + "&service=" + service, &str); - return TString(str.Str()); -} - -NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud) { - NSo::NProto::TDqSolomonShard settings; - settings.SetEndpoint(TStringBuilder() << getenv("SOLOMON_HOST") << ":" << TString(getenv("SOLOMON_PORT"))); - if (isCloud) { - settings.SetProject("folderId1"); - settings.SetCluster("folderId1"); - settings.SetService("custom"); - } else { - settings.SetProject("cloudId1"); - settings.SetCluster("folderId1"); - settings.SetService("custom"); - } - - settings.SetClusterType(isCloud ? NSo::NProto::ESolomonClusterType::CT_MONITORING : NSo::NProto::ESolomonClusterType::CT_SOLOMON); + freeSpace); + + actor.InitSink(dqSink, dqSinkAsActor); + }); +} + +void CleanupSolomon(TString cloudId, TString folderId, TString service, bool isCloud) { + const auto solomonPort = TString(getenv("SOLOMON_PORT")); + TSimpleHttpClient httpClient("localhost", std::stoi(solomonPort)); + TStringStream str; + TStringBuilder builder; + builder << "/cleanup"; + if (isCloud) { + builder << "?folderId=" << folderId << "&service=" << service; + } else { + builder << "?project=" << cloudId << "&cluster=" << folderId << "&service=" << service; + } + + DoWithRetry( + [&]{ httpClient.DoPost(builder, "", &str); }, + TRetryOptions(3, TDuration::Seconds(1)), + true); +} + +TString GetSolomonMetrics(TString folderId, TString service) { + const auto solomonPort = TString(getenv("SOLOMON_PORT")); + TSimpleHttpClient httpClient("localhost", std::stoi(solomonPort)); + TStringStream str; + httpClient.DoGet("/metrics?folderId=" + folderId + "&service=" + service, &str); + return TString(str.Str()); +} + +NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud) { + NSo::NProto::TDqSolomonShard settings; + settings.SetEndpoint(TStringBuilder() << getenv("SOLOMON_HOST") << ":" << TString(getenv("SOLOMON_PORT"))); + if (isCloud) { + settings.SetProject("folderId1"); + settings.SetCluster("folderId1"); + settings.SetService("custom"); + } else { + settings.SetProject("cloudId1"); + settings.SetCluster("folderId1"); + settings.SetService("custom"); + } + + settings.SetClusterType(isCloud ? NSo::NProto::ESolomonClusterType::CT_MONITORING : NSo::NProto::ESolomonClusterType::CT_SOLOMON); settings.SetUseSsl(false); - - FillDqSolomonScheme(*settings.MutableScheme()); - - return settings; -} - -NUdf::TUnboxedValue CreateStruct( - NKikimr::NMiniKQL::THolderFactory& holderFactory, - std::initializer_list<NUdf::TUnboxedValuePod> fields) -{ - NUdf::TUnboxedValue* itemsPtr = nullptr; - auto structValues = holderFactory.CreateDirectArrayHolder(fields.size(), itemsPtr); - for (auto&& field : fields) { - *(itemsPtr++) = std::move(field); - } - return structValues; -} - -int GetMetricsCount(TString metrics) { - NJson::TJsonValue json; - NJson::ReadJsonTree(metrics, &json, true); - return json.GetArray().size(); -} - -} // namespace NYql::NDq + + FillDqSolomonScheme(*settings.MutableScheme()); + + return settings; +} + +NUdf::TUnboxedValue CreateStruct( + NKikimr::NMiniKQL::THolderFactory& holderFactory, + std::initializer_list<NUdf::TUnboxedValuePod> fields) +{ + NUdf::TUnboxedValue* itemsPtr = nullptr; + auto structValues = holderFactory.CreateDirectArrayHolder(fields.size(), itemsPtr); + for (auto&& field : fields) { + *(itemsPtr++) = std::move(field); + } + return structValues; +} + +int GetMetricsCount(TString metrics) { + NJson::TJsonValue json; + NJson::ReadJsonTree(metrics, &json, true); + return json.GetArray().size(); +} + +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h index 9b7b6b57e7c..5ba4931e60e 100644 --- a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h +++ b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h @@ -7,31 +7,31 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> #include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> #include <ydb/library/yql/minikql/mkql_alloc.h> - + #include <ydb/core/testlib/basics/runtime.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <chrono> -#include <queue> - -namespace NYql::NDq { - -void InitSink( - TFakeCASetup& caSetup, - NSo::NProto::TDqSolomonShard&& settings, - i64 freeSpace = 100000); - -void CleanupSolomon(TString cloudId, TString folderId, TString service, bool isCloud); - -TString GetSolomonMetrics(TString folderId, TString service); - -NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud); - -NUdf::TUnboxedValue CreateStruct( - NKikimr::NMiniKQL::THolderFactory& holderFactory, - std::initializer_list<NUdf::TUnboxedValuePod> fields); - -int GetMetricsCount(TString metrics); - -} // namespace NYql::NDq + +#include <library/cpp/testing/unittest/registar.h> + +#include <chrono> +#include <queue> + +namespace NYql::NDq { + +void InitSink( + TFakeCASetup& caSetup, + NSo::NProto::TDqSolomonShard&& settings, + i64 freeSpace = 100000); + +void CleanupSolomon(TString cloudId, TString folderId, TString service, bool isCloud); + +TString GetSolomonMetrics(TString folderId, TString service); + +NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud); + +NUdf::TUnboxedValue CreateStruct( + NKikimr::NMiniKQL::THolderFactory& holderFactory, + std::initializer_list<NUdf::TUnboxedValuePod> fields); + +int GetMetricsCount(TString metrics); + +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/solomon/async_io/ut/ya.make b/ydb/library/yql/providers/solomon/async_io/ut/ya.make index a9aa1845ed7..110566a0027 100644 --- a/ydb/library/yql/providers/solomon/async_io/ut/ya.make +++ b/ydb/library/yql/providers/solomon/async_io/ut/ya.make @@ -1,21 +1,21 @@ UNITTEST_FOR(ydb/library/yql/providers/solomon/async_io) - -OWNER( - d-mokhnatkin + +OWNER( + d-mokhnatkin g:yq - g:yql -) - + g:yql +) + INCLUDE(${ARCADIA_ROOT}/kikimr/yq/tools/solomon_emulator/recipe/recipe.inc) - -SRCS( - dq_solomon_write_actor_ut.cpp - ut_helpers.cpp -) - -PEERDIR( - library/cpp/http/simple - library/cpp/retry + +SRCS( + dq_solomon_write_actor_ut.cpp + ut_helpers.cpp +) + +PEERDIR( + library/cpp/http/simple + library/cpp/retry ydb/core/testlib/basics ydb/library/yql/minikql ydb/library/yql/minikql/computation @@ -23,10 +23,10 @@ PEERDIR( ydb/library/yql/sql ydb/library/yql/providers/common/comp_nodes ydb/library/yql/providers/common/ut_helpers -) - -YQL_LAST_ABI_VERSION() - +) + +YQL_LAST_ABI_VERSION() + REQUIREMENTS(ram:12) -END() +END() diff --git a/ydb/library/yql/providers/solomon/async_io/ya.make b/ydb/library/yql/providers/solomon/async_io/ya.make index 4f1edaa5e3f..ea04c723e73 100644 --- a/ydb/library/yql/providers/solomon/async_io/ya.make +++ b/ydb/library/yql/providers/solomon/async_io/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( +LIBRARY() + +OWNER( g:yq - g:yql -) - -SRCS( - dq_solomon_write_actor.cpp - metrics_encoder.cpp -) - -PEERDIR( - library/cpp/json/easy_parse - library/cpp/monlib/encode/json + g:yql +) + +SRCS( + dq_solomon_write_actor.cpp + metrics_encoder.cpp +) + +PEERDIR( + library/cpp/json/easy_parse + library/cpp/monlib/encode/json ydb/library/yql/minikql/computation ydb/library/yql/providers/common/token_accessor/client ydb/library/yql/public/types @@ -20,12 +20,12 @@ PEERDIR( ydb/library/yql/utils/log ydb/library/yql/dq/actors/compute ydb/library/yql/providers/solomon/proto -) - -YQL_LAST_ABI_VERSION() - -END() - +) + +YQL_LAST_ABI_VERSION() + +END() + IF (NOT OPENSOURCE) IF (OS_LINUX) # Solomon recipe is supported only for linux. @@ -33,5 +33,5 @@ IF (NOT OPENSOURCE) ut ) ENDIF() -ENDIF() +ENDIF() diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json index 1168b665bbb..d064daa1b40 100644 --- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json +++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json @@ -63,28 +63,28 @@ {"Index": 2, "Name": "Shard", "Type": "TCoAtom"}, {"Index": 3, "Name": "Input", "Type": "TExprBase"} ] - }, - { - "Name": "TSoShard", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "SoShard"}, - "Children": [ - {"Index": 0, "Name": "SolomonCluster", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Project", "Type": "TCoAtom"}, - {"Index": 2, "Name": "Cluster", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Service", "Type": "TCoAtom"}, + }, + { + "Name": "TSoShard", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "SoShard"}, + "Children": [ + {"Index": 0, "Name": "SolomonCluster", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Project", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Cluster", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Service", "Type": "TCoAtom"}, {"Index": 4, "Name": "Token", "Type": "TCoSecureParam", "Optional": true} - ] - }, - { - "Name": "TDqSoShardSink", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "DqSoShardSink"}, - "Children": [ - {"Index": 0, "Name": "Shard", "Type": "TSoShard"}, - {"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"}, - {"Index": 2, "Name": "Token", "Type": "TCoSecureParam", "Optional": true} - ] + ] + }, + { + "Name": "TDqSoShardSink", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "DqSoShardSink"}, + "Children": [ + {"Index": 0, "Name": "Shard", "Type": "TSoShard"}, + {"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"}, + {"Index": 2, "Name": "Token", "Type": "TCoSecureParam", "Optional": true} + ] } ] } diff --git a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto index bd1ac45bf25..3bb9d19d51d 100644 --- a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto +++ b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto @@ -1,41 +1,41 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package NYql.NSo.NProto; - -enum ESolomonClusterType { - CT_UNSPECIFIED = 0; - CT_SOLOMON = 1; - CT_MONITORING = 2; -} - +syntax = "proto3"; +option cc_enable_arenas = true; + +package NYql.NSo.NProto; + +enum ESolomonClusterType { + CT_UNSPECIFIED = 0; + CT_SOLOMON = 1; + CT_MONITORING = 2; +} + message TToken { - string Name = 1; -} - -message TDqSolomonSchemeItem { - string Key = 1; - uint32 Index = 2; - uint32 DataTypeId = 3; -} - -message TDqSolomonShardScheme { - TDqSolomonSchemeItem Timestamp = 1; - repeated TDqSolomonSchemeItem Labels = 2; - repeated TDqSolomonSchemeItem Sensors = 3; -}; - -message TDqSolomonShard { - string Endpoint = 1; - string Project = 2; - string Cluster = 3; - string Service = 4; - - ESolomonClusterType ClusterType = 20; + string Name = 1; +} + +message TDqSolomonSchemeItem { + string Key = 1; + uint32 Index = 2; + uint32 DataTypeId = 3; +} + +message TDqSolomonShardScheme { + TDqSolomonSchemeItem Timestamp = 1; + repeated TDqSolomonSchemeItem Labels = 2; + repeated TDqSolomonSchemeItem Sensors = 3; +}; + +message TDqSolomonShard { + string Endpoint = 1; + string Project = 2; + string Cluster = 3; + string Service = 4; + + ESolomonClusterType ClusterType = 20; bool UseSsl = 21; - - TDqSolomonShardScheme Scheme = 30; - - string ServiceAccount = 40; + + TDqSolomonShardScheme Scheme = 30; + + string ServiceAccount = 40; TToken Token = 41; -} +} diff --git a/ydb/library/yql/providers/solomon/proto/ya.make b/ydb/library/yql/providers/solomon/proto/ya.make index 26362702196..b30b1d5cfd0 100644 --- a/ydb/library/yql/providers/solomon/proto/ya.make +++ b/ydb/library/yql/providers/solomon/proto/ya.make @@ -1,14 +1,14 @@ -PROTO_LIBRARY() - -OWNER( +PROTO_LIBRARY() + +OWNER( g:yq - g:yql -) - -SRCS( - dq_solomon_shard.proto -) - -EXCLUDE_TAGS(GO_PROTO) - -END() + g:yql +) + +SRCS( + dq_solomon_shard.proto +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/ydb/library/yql/providers/solomon/provider/ya.make b/ydb/library/yql/providers/solomon/provider/ya.make index c81aa05a3df..b80c01bdb74 100644 --- a/ydb/library/yql/providers/solomon/provider/ya.make +++ b/ydb/library/yql/providers/solomon/provider/ya.make @@ -7,21 +7,21 @@ OWNER( SRCS( yql_solomon_config.cpp - yql_solomon_datasink_execution.cpp - yql_solomon_datasink_type_ann.cpp - yql_solomon_datasink.cpp - yql_solomon_datasource_execution.cpp - yql_solomon_datasource_type_ann.cpp - yql_solomon_datasource.cpp - yql_solomon_dq_integration.cpp + yql_solomon_datasink_execution.cpp + yql_solomon_datasink_type_ann.cpp + yql_solomon_datasink.cpp + yql_solomon_datasource_execution.cpp + yql_solomon_datasource_type_ann.cpp + yql_solomon_datasource.cpp + yql_solomon_dq_integration.cpp yql_solomon_io_discovery.cpp yql_solomon_load_meta.cpp - yql_solomon_physical_optimize.cpp - yql_solomon_provider.cpp + yql_solomon_physical_optimize.cpp + yql_solomon_provider.cpp ) PEERDIR( - library/cpp/actors/protos + library/cpp/actors/protos ydb/library/yql/dq/expr_nodes ydb/library/yql/providers/common/config ydb/library/yql/providers/common/proto diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h index 2c24d3c2a20..941b2adc305 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h @@ -21,13 +21,13 @@ struct TSolomonConfiguration TSolomonConfiguration(const TSolomonConfiguration&) = delete; template <typename TProtoConfig> - void Init(const TProtoConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx) + void Init(const TProtoConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx) { TVector<TString> clusters(Reserve(config.ClusterMappingSize())); for (auto& cluster: config.GetClusterMapping()) { clusters.push_back(cluster.GetName()); - ClusterConfigs[cluster.GetName()] = cluster; - + ClusterConfigs[cluster.GetName()] = cluster; + const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_solomon", cluster.GetToken()); Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken); } @@ -42,9 +42,9 @@ struct TSolomonConfiguration } TSolomonSettings::TConstPtr Snapshot() const; - - THashMap<TString, TSolomonClusterConfig> ClusterConfigs; - THashMap<TString, TString> Tokens; + + THashMap<TString, TSolomonClusterConfig> ClusterConfigs; + THashMap<TString, TString> Tokens; }; } // NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp index 2c8d18ee82f..b09fd381a07 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp @@ -19,7 +19,7 @@ public: : State_ {state} , TypeAnnotationTransformer_(CreateSolomonDataSinkTypeAnnotationTransformer(State_)) , ExecutionTransformer_(CreateSolomonDataSinkExecTransformer(State_)) - , PhysicalOptProposalTransformer_(CreateSoPhysicalOptProposalTransformer(State_)) + , PhysicalOptProposalTransformer_(CreateSoPhysicalOptProposalTransformer(State_)) { } @@ -39,10 +39,10 @@ public: .Done().Ptr(); } - const THashMap<TString, TString>* GetClusterTokens() override { - return &State_->Configuration->Tokens; - } - + const THashMap<TString, TString>* GetClusterTokens() override { + return &State_->Configuration->Tokens; + } + IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { Y_UNUSED(instantOnly); return *TypeAnnotationTransformer_; @@ -52,10 +52,10 @@ public: return *ExecutionTransformer_; } - IGraphTransformer& GetPhysicalOptProposalTransformer() override { - return *PhysicalOptProposalTransformer_; - } - + IGraphTransformer& GetPhysicalOptProposalTransformer() override { + return *PhysicalOptProposalTransformer_; + } + bool CanParse(const TExprNode& node) override { if (node.IsCallable(TCoWrite::CallableName())) { return TSoDataSink::Match(node.Child(1)); @@ -160,16 +160,16 @@ public: return TStringBuilder() << SolomonProviderName << '.' << node.Child(1)->Content(); } - IDqIntegration* GetDqIntegration() override { - return State_->IsRtmrMode() ? nullptr : State_->DqIntegration.Get(); - } - + IDqIntegration* GetDqIntegration() override { + return State_->IsRtmrMode() ? nullptr : State_->DqIntegration.Get(); + } + private: TSolomonState::TPtr State_; THolder<TVisitorTransformerBase> TypeAnnotationTransformer_; THolder<TExecTransformerBase> ExecutionTransformer_; - THolder<IGraphTransformer> PhysicalOptProposalTransformer_; + THolder<IGraphTransformer> PhysicalOptProposalTransformer_; }; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp index 8b4c7945295..e3048498c7b 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp @@ -8,126 +8,126 @@ #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h> #include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h> - + #include <ydb/library/yql/utils/log/log.h> -#include <util/string/split.h> - +#include <util/string/split.h> + namespace NYql { using namespace NNodes; class TSolomonDataSinkExecTransformer : public TExecTransformerBase { public: - explicit TSolomonDataSinkExecTransformer(TSolomonState::TPtr state) + explicit TSolomonDataSinkExecTransformer(TSolomonState::TPtr state) : State_(state) { - AddHandler({TCoCommit::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleCommit)); - AddHandler({TSoWriteToShard::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleSoWriteToShard)); + AddHandler({TCoCommit::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleCommit)); + AddHandler({TSoWriteToShard::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleSoWriteToShard)); + } + + TStatusCallbackPair HandleCommit(const TExprNode::TPtr& input, TExprContext& ctx) { + if (!State_->IsRtmrMode() && TDqQuery::Match(input->Child(TCoCommit::idx_World))) { + return DelegateExecutionToDqProvider(input->ChildPtr(TCoCommit::idx_World), input, ctx); + } + + return ExecPass(input, ctx); + } + + TStatusCallbackPair HandleSoWriteToShard(const TExprNode::TPtr& input, TExprContext& ctx) { + if (State_->IsRtmrMode()) { + return ExecPass(input, ctx); + } + + return DelegateExecutionToDqProvider(input->ChildPtr(TSoWriteToShard::idx_Input), input, ctx); } - TStatusCallbackPair HandleCommit(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!State_->IsRtmrMode() && TDqQuery::Match(input->Child(TCoCommit::idx_World))) { - return DelegateExecutionToDqProvider(input->ChildPtr(TCoCommit::idx_World), input, ctx); - } - - return ExecPass(input, ctx); - } - - TStatusCallbackPair HandleSoWriteToShard(const TExprNode::TPtr& input, TExprContext& ctx) { - if (State_->IsRtmrMode()) { - return ExecPass(input, ctx); - } - - return DelegateExecutionToDqProvider(input->ChildPtr(TSoWriteToShard::idx_Input), input, ctx); - } - private: - TStatusCallbackPair DelegateExecutionToDqProvider( - const TExprNode::TPtr& input, - const TExprNode::TPtr& originInput, - TExprContext& ctx) - { - YQL_CLOG(INFO, ProviderSolomon) << "Delegate execution of " << input->Content() << " to dq provider"; - auto delegatedNode = Build<TPull>(ctx, input->Pos()) - .Input(input) - .BytesLimit() - .Value(TString()) - .Build() - .RowsLimit() - .Value(TString("0")) - .Build() - .FormatDetails() + TStatusCallbackPair DelegateExecutionToDqProvider( + const TExprNode::TPtr& input, + const TExprNode::TPtr& originInput, + TExprContext& ctx) + { + YQL_CLOG(INFO, ProviderSolomon) << "Delegate execution of " << input->Content() << " to dq provider"; + auto delegatedNode = Build<TPull>(ctx, input->Pos()) + .Input(input) + .BytesLimit() + .Value(TString()) + .Build() + .RowsLimit() + .Value(TString("0")) + .Build() + .FormatDetails() .Value(ToString((ui32)NYson::EYsonFormat::Binary)) - .Build() - .Settings() - .Build() - .Format() - .Value(ToString("0")) - .Build() - .PublicId() - .Value("id") - .Build() - .Discard() - .Value(ToString(true)) - .Build() - .Origin(originInput) - .Done() - .Ptr(); - - auto atomType = ctx.MakeType<TUnitExprType>(); - - for (auto idx: {TResOrPullBase::idx_BytesLimit, TResOrPullBase::idx_RowsLimit, TResOrPullBase::idx_FormatDetails, - TResOrPullBase::idx_Format, TResOrPullBase::idx_PublicId, TResOrPullBase::idx_Discard }) { - delegatedNode->Child(idx)->SetTypeAnn(atomType); - delegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete); - } - - delegatedNode->SetTypeAnn(originInput->GetTypeAnn()); - delegatedNode->SetState(TExprNode::EState::ConstrComplete); - originInput->SetState(TExprNode::EState::ExecutionInProgress); - - auto dqProvider = State_->Types->DataSourceMap.FindPtr(DqProviderName); - YQL_ENSURE(dqProvider); - - TExprNode::TPtr delegatedNodeOutput; - auto status = dqProvider->Get()->GetCallableExecutionTransformer().Transform(delegatedNode, delegatedNodeOutput, ctx); - - if (status.Level != TStatus::Async) { - YQL_ENSURE(status.Level != TStatus::Ok, "Asynchronous execution is expected in a happy path."); - return SyncStatus(status); - } - - auto dqFuture = dqProvider->Get()->GetCallableExecutionTransformer().GetAsyncFuture(*delegatedNode); - - TAsyncTransformCallbackFuture callbackFuture = dqFuture.Apply( - [dqProvider, delegatedNode](const NThreading::TFuture<void>& completedFuture) { - return TAsyncTransformCallback( - [completedFuture, dqProvider, delegatedNode](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - completedFuture.GetValue(); - TExprNode::TPtr delegatedNodeOutput; - auto dqWriteStatus = dqProvider->Get()->GetCallableExecutionTransformer() - .ApplyAsyncChanges(delegatedNode, delegatedNodeOutput, ctx); - - YQL_ENSURE(dqWriteStatus != TStatus::Async, "ApplyAsyncChanges should not return Async."); - - if (dqWriteStatus != TStatus::Ok) { - output = input; - return dqWriteStatus; - } - - input->SetState(TExprNode::EState::ExecutionComplete); - output = ctx.ShallowCopy(*input); - output->SetResult(ctx.NewAtom(input->Pos(), "DQ_completed")); - - return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); - }); - }); - - return std::make_pair(IGraphTransformer::TStatus::Async, callbackFuture); - } - -private: + .Build() + .Settings() + .Build() + .Format() + .Value(ToString("0")) + .Build() + .PublicId() + .Value("id") + .Build() + .Discard() + .Value(ToString(true)) + .Build() + .Origin(originInput) + .Done() + .Ptr(); + + auto atomType = ctx.MakeType<TUnitExprType>(); + + for (auto idx: {TResOrPullBase::idx_BytesLimit, TResOrPullBase::idx_RowsLimit, TResOrPullBase::idx_FormatDetails, + TResOrPullBase::idx_Format, TResOrPullBase::idx_PublicId, TResOrPullBase::idx_Discard }) { + delegatedNode->Child(idx)->SetTypeAnn(atomType); + delegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete); + } + + delegatedNode->SetTypeAnn(originInput->GetTypeAnn()); + delegatedNode->SetState(TExprNode::EState::ConstrComplete); + originInput->SetState(TExprNode::EState::ExecutionInProgress); + + auto dqProvider = State_->Types->DataSourceMap.FindPtr(DqProviderName); + YQL_ENSURE(dqProvider); + + TExprNode::TPtr delegatedNodeOutput; + auto status = dqProvider->Get()->GetCallableExecutionTransformer().Transform(delegatedNode, delegatedNodeOutput, ctx); + + if (status.Level != TStatus::Async) { + YQL_ENSURE(status.Level != TStatus::Ok, "Asynchronous execution is expected in a happy path."); + return SyncStatus(status); + } + + auto dqFuture = dqProvider->Get()->GetCallableExecutionTransformer().GetAsyncFuture(*delegatedNode); + + TAsyncTransformCallbackFuture callbackFuture = dqFuture.Apply( + [dqProvider, delegatedNode](const NThreading::TFuture<void>& completedFuture) { + return TAsyncTransformCallback( + [completedFuture, dqProvider, delegatedNode](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + completedFuture.GetValue(); + TExprNode::TPtr delegatedNodeOutput; + auto dqWriteStatus = dqProvider->Get()->GetCallableExecutionTransformer() + .ApplyAsyncChanges(delegatedNode, delegatedNodeOutput, ctx); + + YQL_ENSURE(dqWriteStatus != TStatus::Async, "ApplyAsyncChanges should not return Async."); + + if (dqWriteStatus != TStatus::Ok) { + output = input; + return dqWriteStatus; + } + + input->SetState(TExprNode::EState::ExecutionComplete); + output = ctx.ShallowCopy(*input); + output->SetResult(ctx.NewAtom(input->Pos(), "DQ_completed")); + + return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); + }); + }); + + return std::make_pair(IGraphTransformer::TStatus::Async, callbackFuture); + } + +private: TSolomonState::TPtr State_; }; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp index b0043b9cd32..dc9ea45a8f6 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp @@ -14,7 +14,7 @@ public: { using TSelf = TSolomonDataSinkTypeAnnotationTransformer; AddHandler({TSoWriteToShard::CallableName()}, Hndl(&TSelf::HandleWriteToShard)); - AddHandler({TSoShard::CallableName()}, Hndl(&TSelf::HandleSoShard)); + AddHandler({TSoShard::CallableName()}, Hndl(&TSelf::HandleSoShard)); AddHandler({TCoCommit::CallableName()}, Hndl(&TSelf::HandleCommit)); } @@ -37,39 +37,39 @@ private: return TStatus::Ok; } - TStatus HandleSoShard(TExprBase input, TExprContext& ctx) { - YQL_ENSURE(!State_->IsRtmrMode(), "SoShard can't be used in rtmr mode"); - - if (!EnsureMinArgsCount(input.Ref(), 4, ctx) || !EnsureMaxArgsCount(input.Ref(), 5, ctx)) { - return TStatus::Error; - } - - const TSoShard shard = input.Cast<TSoShard>(); - - if (!EnsureAtom(shard.SolomonCluster().Ref(), ctx)) { - return TStatus::Error; - } - - if (!EnsureAtom(shard.Project().Ref(), ctx)) { - return TStatus::Error; - } - - if (!EnsureAtom(shard.Cluster().Ref(), ctx)) { - return TStatus::Error; - } - - if (!EnsureAtom(shard.Service().Ref(), ctx)) { - return TStatus::Error; - } - - if (shard.Token() && !EnsureCallable(shard.Token().Ref(), ctx)) { - return TStatus::Error; - } - - input.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>()); - return TStatus::Ok; - } - + TStatus HandleSoShard(TExprBase input, TExprContext& ctx) { + YQL_ENSURE(!State_->IsRtmrMode(), "SoShard can't be used in rtmr mode"); + + if (!EnsureMinArgsCount(input.Ref(), 4, ctx) || !EnsureMaxArgsCount(input.Ref(), 5, ctx)) { + return TStatus::Error; + } + + const TSoShard shard = input.Cast<TSoShard>(); + + if (!EnsureAtom(shard.SolomonCluster().Ref(), ctx)) { + return TStatus::Error; + } + + if (!EnsureAtom(shard.Project().Ref(), ctx)) { + return TStatus::Error; + } + + if (!EnsureAtom(shard.Cluster().Ref(), ctx)) { + return TStatus::Error; + } + + if (!EnsureAtom(shard.Service().Ref(), ctx)) { + return TStatus::Error; + } + + if (shard.Token() && !EnsureCallable(shard.Token().Ref(), ctx)) { + return TStatus::Error; + } + + input.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>()); + return TStatus::Ok; + } + TStatus HandleCommit(TExprBase input, TExprContext& ctx) { Y_UNUSED(ctx); auto commit = input.Cast<TCoCommit>(); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index c93cbd3cd2d..773c9609914 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -1,5 +1,5 @@ -#include "yql_solomon_dq_integration.h" - +#include "yql_solomon_dq_integration.h" + #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h> #include <ydb/library/yql/utils/log/log.h> @@ -9,150 +9,150 @@ #include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> #include <ydb/library/yql/providers/solomon/proto/dq_solomon_shard.pb.h> - -#include <util/string/builder.h> - -namespace NYql { - -using namespace NNodes; - -namespace { - -constexpr i32 MaxLabelsCount = 16; -constexpr i32 MaxSensorsCount = 50; - -NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) { - switch (clusterType) { - case TSolomonClusterConfig::SCT_SOLOMON: - return NSo::NProto::ESolomonClusterType::CT_SOLOMON; - case TSolomonClusterConfig::SCT_MONITORING: - return NSo::NProto::ESolomonClusterType::CT_MONITORING; - default: - YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterType)); - } -} - + +#include <util/string/builder.h> + +namespace NYql { + +using namespace NNodes; + +namespace { + +constexpr i32 MaxLabelsCount = 16; +constexpr i32 MaxSensorsCount = 50; + +NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) { + switch (clusterType) { + case TSolomonClusterConfig::SCT_SOLOMON: + return NSo::NProto::ESolomonClusterType::CT_SOLOMON; + case TSolomonClusterConfig::SCT_MONITORING: + return NSo::NProto::ESolomonClusterType::CT_MONITORING; + default: + YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterType)); + } +} + const TTypeAnnotationNode* GetItemType(const TExprNode& node) { - const TTypeAnnotationNode* typeAnn = node.GetTypeAnn(); - switch (typeAnn->GetKind()) { - case ETypeAnnotationKind::Flow: + const TTypeAnnotationNode* typeAnn = node.GetTypeAnn(); + switch (typeAnn->GetKind()) { + case ETypeAnnotationKind::Flow: return typeAnn->Cast<TFlowExprType>()->GetItemType(); - case ETypeAnnotationKind::Stream: + case ETypeAnnotationKind::Stream: return typeAnn->Cast<TStreamExprType>()->GetItemType(); - default: break; - } - YQL_ENSURE(false, "Invalid solomon sink type " << typeAnn->GetKind()); + default: break; + } + YQL_ENSURE(false, "Invalid solomon sink type " << typeAnn->GetKind()); return nullptr; -} - -void FillScheme(const TTypeAnnotationNode& itemType, NSo::NProto::TDqSolomonShardScheme& scheme) { - int index = 0; - for (const TItemExprType* structItem : itemType.Cast<TStructExprType>()->GetItems()) { - const auto itemName = structItem->GetName(); - const auto* itemType = structItem->GetItemType(); - YQL_ENSURE( - itemType->GetKind() != ETypeAnnotationKind::Optional, - "Optional types are not supported in monitoring sink. FieldName: " << itemName); - - const auto dataType = NUdf::GetDataTypeInfo(itemType->Cast<TDataExprType>()->GetSlot()); - - NSo::NProto::TDqSolomonSchemeItem schemeItem; - schemeItem.SetKey(TString(itemName)); - schemeItem.SetIndex(index++); - schemeItem.SetDataTypeId(dataType.TypeId); - - if (dataType.Features & NUdf::DateType || dataType.Features & NUdf::TzDateType) { - YQL_ENSURE(!scheme.HasTimestamp(), "Multiple timestamps were provided for monitoing sink"); - *scheme.MutableTimestamp() = schemeItem; - } else if (dataType.Features & NUdf::NumericType) { - scheme.MutableSensors()->Add(std::move(schemeItem)); - } else if (dataType.Features & NUdf::StringType) { - scheme.MutableLabels()->Add(std::move(schemeItem)); - } else { - YQL_ENSURE(false, "Ivalid data type for monitoing sink: " << dataType.Name); - } - } - - YQL_ENSURE(scheme.HasTimestamp(), "Timestamp wasn't provided for monitoing sink"); - YQL_ENSURE(!scheme.GetSensors().empty(), "No sensors were provided for monitoing sink"); - YQL_ENSURE(!scheme.GetLabels().empty(), "No labels were provided for monitoing sink"); - - YQL_ENSURE(scheme.GetLabels().size() <= MaxLabelsCount, - "Max labels count is " << MaxLabelsCount << " but " << scheme.GetLabels().size() << " were provided"); - YQL_ENSURE(scheme.GetSensors().size() <= MaxSensorsCount, - "Max sensors count is " << MaxSensorsCount << " but " << scheme.GetSensors().size() << " were provided"); -} - -class TSolomonDqIntegration: public TDqIntegrationBase { -public: - TSolomonDqIntegration(const TSolomonState::TPtr& state) - : State_(state.Get()) - { - } - - TMaybe<ui64> CanRead(const TDqSettings&, const TExprNode&, TExprContext&, bool) override { - YQL_ENSURE(false, "Unimplemented"); - } - - TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr&, TExprContext&) override { - YQL_ENSURE(false, "Unimplemented"); - } - +} + +void FillScheme(const TTypeAnnotationNode& itemType, NSo::NProto::TDqSolomonShardScheme& scheme) { + int index = 0; + for (const TItemExprType* structItem : itemType.Cast<TStructExprType>()->GetItems()) { + const auto itemName = structItem->GetName(); + const auto* itemType = structItem->GetItemType(); + YQL_ENSURE( + itemType->GetKind() != ETypeAnnotationKind::Optional, + "Optional types are not supported in monitoring sink. FieldName: " << itemName); + + const auto dataType = NUdf::GetDataTypeInfo(itemType->Cast<TDataExprType>()->GetSlot()); + + NSo::NProto::TDqSolomonSchemeItem schemeItem; + schemeItem.SetKey(TString(itemName)); + schemeItem.SetIndex(index++); + schemeItem.SetDataTypeId(dataType.TypeId); + + if (dataType.Features & NUdf::DateType || dataType.Features & NUdf::TzDateType) { + YQL_ENSURE(!scheme.HasTimestamp(), "Multiple timestamps were provided for monitoing sink"); + *scheme.MutableTimestamp() = schemeItem; + } else if (dataType.Features & NUdf::NumericType) { + scheme.MutableSensors()->Add(std::move(schemeItem)); + } else if (dataType.Features & NUdf::StringType) { + scheme.MutableLabels()->Add(std::move(schemeItem)); + } else { + YQL_ENSURE(false, "Ivalid data type for monitoing sink: " << dataType.Name); + } + } + + YQL_ENSURE(scheme.HasTimestamp(), "Timestamp wasn't provided for monitoing sink"); + YQL_ENSURE(!scheme.GetSensors().empty(), "No sensors were provided for monitoing sink"); + YQL_ENSURE(!scheme.GetLabels().empty(), "No labels were provided for monitoing sink"); + + YQL_ENSURE(scheme.GetLabels().size() <= MaxLabelsCount, + "Max labels count is " << MaxLabelsCount << " but " << scheme.GetLabels().size() << " were provided"); + YQL_ENSURE(scheme.GetSensors().size() <= MaxSensorsCount, + "Max sensors count is " << MaxSensorsCount << " but " << scheme.GetSensors().size() << " were provided"); +} + +class TSolomonDqIntegration: public TDqIntegrationBase { +public: + TSolomonDqIntegration(const TSolomonState::TPtr& state) + : State_(state.Get()) + { + } + + TMaybe<ui64> CanRead(const TDqSettings&, const TExprNode&, TExprContext&, bool) override { + YQL_ENSURE(false, "Unimplemented"); + } + + TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr&, TExprContext&) override { + YQL_ENSURE(false, "Unimplemented"); + } + TMaybe<bool> CanWrite(const TDqSettings&, const TExprNode&, TExprContext&) override { - YQL_ENSURE(false, "Unimplemented"); - } - - void FillSourceSettings(const TExprNode&, ::google::protobuf::Any&, TString& ) override { - YQL_ENSURE(false, "Unimplemented"); - } - - void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& protoSettings, TString& sinkType) override { - const auto maybeDqSink = TMaybeNode<TDqSink>(&node); - if (!maybeDqSink) { - return; - } - const auto dqSink = maybeDqSink.Cast(); - - const auto settings = dqSink.Settings(); - const auto maybeShard = TMaybeNode<TSoShard>(settings.Raw()); - if (!maybeShard) { - return; - } - - const TSoShard shard = maybeShard.Cast(); - - const auto solomonCluster = shard.SolomonCluster().StringValue(); - const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(solomonCluster); - YQL_ENSURE(clusterDesc, "Unknown cluster " << solomonCluster); - - NSo::NProto::TDqSolomonShard shardDesc; - shardDesc.SetEndpoint(clusterDesc->GetCluster()); - shardDesc.SetProject(shard.Project().StringValue()); - shardDesc.SetCluster(shard.Cluster().StringValue()); - shardDesc.SetService(shard.Service().StringValue()); - - shardDesc.SetClusterType(MapClusterType(clusterDesc->GetClusterType())); + YQL_ENSURE(false, "Unimplemented"); + } + + void FillSourceSettings(const TExprNode&, ::google::protobuf::Any&, TString& ) override { + YQL_ENSURE(false, "Unimplemented"); + } + + void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& protoSettings, TString& sinkType) override { + const auto maybeDqSink = TMaybeNode<TDqSink>(&node); + if (!maybeDqSink) { + return; + } + const auto dqSink = maybeDqSink.Cast(); + + const auto settings = dqSink.Settings(); + const auto maybeShard = TMaybeNode<TSoShard>(settings.Raw()); + if (!maybeShard) { + return; + } + + const TSoShard shard = maybeShard.Cast(); + + const auto solomonCluster = shard.SolomonCluster().StringValue(); + const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(solomonCluster); + YQL_ENSURE(clusterDesc, "Unknown cluster " << solomonCluster); + + NSo::NProto::TDqSolomonShard shardDesc; + shardDesc.SetEndpoint(clusterDesc->GetCluster()); + shardDesc.SetProject(shard.Project().StringValue()); + shardDesc.SetCluster(shard.Cluster().StringValue()); + shardDesc.SetService(shard.Service().StringValue()); + + shardDesc.SetClusterType(MapClusterType(clusterDesc->GetClusterType())); shardDesc.SetUseSsl(clusterDesc->GetUseSsl()); - + const TTypeAnnotationNode* itemType = GetItemType(node); - FillScheme(*itemType, *shardDesc.MutableScheme()); - + FillScheme(*itemType, *shardDesc.MutableScheme()); + if (auto maybeToken = shard.Token()) { shardDesc.MutableToken()->SetName(TString(maybeToken.Cast().Name().Value())); - } - - protoSettings.PackFrom(shardDesc); - sinkType = "SolomonSink"; - } - -private: - TSolomonState* State_; // State owns dq integration, so back reference must be not smart. -}; - -} - -THolder<IDqIntegration> CreateSolomonDqIntegration(const TSolomonState::TPtr& state) { + } + + protoSettings.PackFrom(shardDesc); + sinkType = "SolomonSink"; + } + +private: + TSolomonState* State_; // State owns dq integration, so back reference must be not smart. +}; + +} + +THolder<IDqIntegration> CreateSolomonDqIntegration(const TSolomonState::TPtr& state) { return MakeHolder<TSolomonDqIntegration>(state); -} - -} +} + +} diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h index 3355341af97..7a621caff16 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h @@ -1,13 +1,13 @@ -#pragma once - -#include "yql_solomon_provider.h" - +#pragma once + +#include "yql_solomon_provider.h" + #include <ydb/library/yql/providers/dq/interface/yql_dq_integration.h> - -#include <util/generic/ptr.h> - -namespace NYql { - -THolder<IDqIntegration> CreateSolomonDqIntegration(const TSolomonState::TPtr& state); - -} + +#include <util/generic/ptr.h> + +namespace NYql { + +THolder<IDqIntegration> CreateSolomonDqIntegration(const TSolomonState::TPtr& state); + +} diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp index ad60fc7b378..226205cce20 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp @@ -1,5 +1,5 @@ -#include "yql_solomon_provider_impl.h" - +#include "yql_solomon_provider_impl.h" + #include <ydb/library/yql/core/yql_opt_utils.h> #include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h> #include <ydb/library/yql/dq/opt/dq_opt.h> @@ -9,168 +9,168 @@ #include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h> #include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h> #include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h> - -#include <util/string/split.h> - -namespace NYql { - -namespace { - -using namespace NNodes; -using namespace NDq; - -TString FormatShardPathInvalid(const TSolomonClusterConfig& clusterConfig, const TString& path) { - TStringBuilder err; - err << "Invalid shard path " << path << " It should be "; - switch (clusterConfig.GetClusterType()) { - case TSolomonClusterConfig::SCT_SOLOMON: - err << "{project}/{cluster}/{service}"; - break; - case TSolomonClusterConfig::SCT_MONITORING: - err << "{cloudId}/{folderId}/{service}"; - break; - default: - YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterConfig.GetClusterType())); - } - err << " or just {service} when using connection."; - return err; -} - -void ParseShardPath( - const TSolomonClusterConfig& clusterConfig, - const TString& path, - TString& project, - TString& cluster, - TString& service) -{ - std::vector<TString> shardData; - shardData.reserve(3); - for (const auto& it : StringSplitter(path).Split('/')) { - shardData.emplace_back(it.Token()); - } - YQL_ENSURE(shardData.size() == 1 || shardData.size() == 3, "" << FormatShardPathInvalid(clusterConfig, path)); - - project = shardData.size() == 3 ? shardData.at(0) : TString(); - cluster = shardData.size() == 3 ? shardData.at(1) : TString(); - service = shardData.back(); -} - -class TSoPhysicalOptProposalTransformer : public TOptimizeTransformerBase { -public: - explicit TSoPhysicalOptProposalTransformer(TSolomonState::TPtr state) - : TOptimizeTransformerBase(state->Types, NLog::EComponent::ProviderSolomon, {}) - , State_(std::move(state)) - { -#define HNDL(name) "PhysicalOptimizer-"#name, Hndl(&TSoPhysicalOptProposalTransformer::name) - AddHandler(0, &TSoWriteToShard::Match, HNDL(SoWriteToShard)); -#undef HNDL + +#include <util/string/split.h> + +namespace NYql { + +namespace { + +using namespace NNodes; +using namespace NDq; + +TString FormatShardPathInvalid(const TSolomonClusterConfig& clusterConfig, const TString& path) { + TStringBuilder err; + err << "Invalid shard path " << path << " It should be "; + switch (clusterConfig.GetClusterType()) { + case TSolomonClusterConfig::SCT_SOLOMON: + err << "{project}/{cluster}/{service}"; + break; + case TSolomonClusterConfig::SCT_MONITORING: + err << "{cloudId}/{folderId}/{service}"; + break; + default: + YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterConfig.GetClusterType())); + } + err << " or just {service} when using connection."; + return err; +} + +void ParseShardPath( + const TSolomonClusterConfig& clusterConfig, + const TString& path, + TString& project, + TString& cluster, + TString& service) +{ + std::vector<TString> shardData; + shardData.reserve(3); + for (const auto& it : StringSplitter(path).Split('/')) { + shardData.emplace_back(it.Token()); + } + YQL_ENSURE(shardData.size() == 1 || shardData.size() == 3, "" << FormatShardPathInvalid(clusterConfig, path)); + + project = shardData.size() == 3 ? shardData.at(0) : TString(); + cluster = shardData.size() == 3 ? shardData.at(1) : TString(); + service = shardData.back(); +} + +class TSoPhysicalOptProposalTransformer : public TOptimizeTransformerBase { +public: + explicit TSoPhysicalOptProposalTransformer(TSolomonState::TPtr state) + : TOptimizeTransformerBase(state->Types, NLog::EComponent::ProviderSolomon, {}) + , State_(std::move(state)) + { +#define HNDL(name) "PhysicalOptimizer-"#name, Hndl(&TSoPhysicalOptProposalTransformer::name) + AddHandler(0, &TSoWriteToShard::Match, HNDL(SoWriteToShard)); +#undef HNDL SetGlobal(0); // Stage 0 of this optimizer is global => we can remap nodes. - } - - TMaybeNode<TExprBase> SoWriteToShard(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) const { - if (State_->IsRtmrMode()) { - return node; - } - - auto write = node.Cast<TSoWriteToShard>(); - if (!TDqCnUnionAll::Match(write.Input().Raw())) { - // If input is not DqCnUnionAll, it means not all dq optimizations are done yet - return node; - } - - const TParentsMap* parentsMap = getParents(); - auto dqUnion = write.Input().Cast<TDqCnUnionAll>(); - if (!NDq::IsSingleConsumerConnection(dqUnion, *parentsMap)) { - return node; - } - - YQL_CLOG(INFO, ProviderSolomon) << "Optimize SoWriteToShard"; - - const auto solomonCluster = TString(write.DataSink().Cluster().Value()); - auto shard = BuildSolomonShard(write.Shard().Cast<TCoAtom>(), ctx, solomonCluster); - - auto dqSink = Build<TDqSink>(ctx, write.Pos()) - .DataSink(write.DataSink()) - .Settings(shard) - .Index(dqUnion.Output().Index()) - .Done(); - - TDqStage inputStage = dqUnion.Output().Stage().Cast<TDqStage>(); - + } + + TMaybeNode<TExprBase> SoWriteToShard(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) const { + if (State_->IsRtmrMode()) { + return node; + } + + auto write = node.Cast<TSoWriteToShard>(); + if (!TDqCnUnionAll::Match(write.Input().Raw())) { + // If input is not DqCnUnionAll, it means not all dq optimizations are done yet + return node; + } + + const TParentsMap* parentsMap = getParents(); + auto dqUnion = write.Input().Cast<TDqCnUnionAll>(); + if (!NDq::IsSingleConsumerConnection(dqUnion, *parentsMap)) { + return node; + } + + YQL_CLOG(INFO, ProviderSolomon) << "Optimize SoWriteToShard"; + + const auto solomonCluster = TString(write.DataSink().Cluster().Value()); + auto shard = BuildSolomonShard(write.Shard().Cast<TCoAtom>(), ctx, solomonCluster); + + auto dqSink = Build<TDqSink>(ctx, write.Pos()) + .DataSink(write.DataSink()) + .Settings(shard) + .Index(dqUnion.Output().Index()) + .Done(); + + TDqStage inputStage = dqUnion.Output().Stage().Cast<TDqStage>(); + auto sinksBuilder = Build<TDqSinksList>(ctx, inputStage.Pos()); if (inputStage.Sinks()) { sinksBuilder.InitFrom(inputStage.Sinks().Cast()); } sinksBuilder.Add(dqSink); - + auto dqStageWithSink = Build<TDqStage>(ctx, inputStage.Pos()) .InitFrom(inputStage) .Sinks(sinksBuilder.Done()) .Done(); - - auto dqQueryBuilder = Build<TDqQuery>(ctx, write.Pos()); - dqQueryBuilder.World(write.World()); - dqQueryBuilder.SinkStages().Add(dqStageWithSink).Build(); + + auto dqQueryBuilder = Build<TDqQuery>(ctx, write.Pos()); + dqQueryBuilder.World(write.World()); + dqQueryBuilder.SinkStages().Add(dqStageWithSink).Build(); optCtx.RemapNode(inputStage.Ref(), dqStageWithSink.Ptr()); - return dqQueryBuilder.Done(); - } - -private: - TExprBase BuildSolomonShard(TCoAtom shardNode, TExprContext& ctx, TString solomonCluster) const { - const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(solomonCluster); - YQL_ENSURE(clusterDesc, "Unknown cluster " << solomonCluster); - - TString project, cluster, service; - ParseShardPath(*clusterDesc, shardNode.StringValue(), project, cluster, service); - - if (project.empty() && clusterDesc->HasPath()) { - project = clusterDesc->GetPath().GetProject(); - } - YQL_ENSURE(!project.empty(), "Project is not defined. You can define it inside connection, or inside query."); - - if (cluster.empty() && clusterDesc->HasPath()) { - cluster = clusterDesc->GetPath().GetCluster(); - } - YQL_ENSURE(!cluster.empty(), "Cluster is not defined. You can define it inside connection, or inside query."); - - auto solomonClusterAtom = Build<TCoAtom>(ctx, shardNode.Pos()) - .Value(solomonCluster) - .Done(); - - auto projectAtom = Build<TCoAtom>(ctx, shardNode.Pos()) - .Value(project) - .Done(); - - auto clusterAtom = Build<TCoAtom>(ctx, shardNode.Pos()) - .Value(cluster) - .Done(); - - auto serviceAtom = Build<TCoAtom>(ctx, shardNode.Pos()) - .Value(service) - .Done(); - - auto dqSoShardBuilder = Build<TSoShard>(ctx, shardNode.Pos()); - dqSoShardBuilder.SolomonCluster(solomonClusterAtom); - dqSoShardBuilder.Project(projectAtom); - dqSoShardBuilder.Cluster(clusterAtom); - dqSoShardBuilder.Service(serviceAtom); - - dqSoShardBuilder.Token<TCoSecureParam>().Name().Build("cluster:default_" + solomonCluster).Build(); - - return dqSoShardBuilder.Done(); - } - -private: - TSolomonState::TPtr State_; -}; - -} // namespace - -THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state) { - return MakeHolder<TSoPhysicalOptProposalTransformer>(std::move(state)); -} - -} // namespace NYql + return dqQueryBuilder.Done(); + } + +private: + TExprBase BuildSolomonShard(TCoAtom shardNode, TExprContext& ctx, TString solomonCluster) const { + const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(solomonCluster); + YQL_ENSURE(clusterDesc, "Unknown cluster " << solomonCluster); + + TString project, cluster, service; + ParseShardPath(*clusterDesc, shardNode.StringValue(), project, cluster, service); + + if (project.empty() && clusterDesc->HasPath()) { + project = clusterDesc->GetPath().GetProject(); + } + YQL_ENSURE(!project.empty(), "Project is not defined. You can define it inside connection, or inside query."); + + if (cluster.empty() && clusterDesc->HasPath()) { + cluster = clusterDesc->GetPath().GetCluster(); + } + YQL_ENSURE(!cluster.empty(), "Cluster is not defined. You can define it inside connection, or inside query."); + + auto solomonClusterAtom = Build<TCoAtom>(ctx, shardNode.Pos()) + .Value(solomonCluster) + .Done(); + + auto projectAtom = Build<TCoAtom>(ctx, shardNode.Pos()) + .Value(project) + .Done(); + + auto clusterAtom = Build<TCoAtom>(ctx, shardNode.Pos()) + .Value(cluster) + .Done(); + + auto serviceAtom = Build<TCoAtom>(ctx, shardNode.Pos()) + .Value(service) + .Done(); + + auto dqSoShardBuilder = Build<TSoShard>(ctx, shardNode.Pos()); + dqSoShardBuilder.SolomonCluster(solomonClusterAtom); + dqSoShardBuilder.Project(projectAtom); + dqSoShardBuilder.Cluster(clusterAtom); + dqSoShardBuilder.Service(serviceAtom); + + dqSoShardBuilder.Token<TCoSecureParam>().Name().Build("cluster:default_" + solomonCluster).Build(); + + return dqSoShardBuilder.Done(); + } + +private: + TSolomonState::TPtr State_; +}; + +} // namespace + +THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state) { + return MakeHolder<TSoPhysicalOptProposalTransformer>(std::move(state)); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp index 7db0ba0cc69..d84f1502a65 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp @@ -6,8 +6,8 @@ namespace NYql { -TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode) { - return [gateway, supportRtmrMode] ( +TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode) { + return [gateway, supportRtmrMode] ( const TString& userName, const TString& sessionId, const TGatewaysConfig* gatewaysConfig, @@ -26,12 +26,12 @@ TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr auto solomonState = MakeIntrusive<TSolomonState>(); - solomonState->SupportRtmrMode = supportRtmrMode; + solomonState->SupportRtmrMode = supportRtmrMode; solomonState->Types = typeCtx.Get(); solomonState->Gateway = gateway; - solomonState->DqIntegration = CreateSolomonDqIntegration(solomonState); + solomonState->DqIntegration = CreateSolomonDqIntegration(solomonState); if (gatewaysConfig) { - solomonState->Configuration->Init(gatewaysConfig->GetSolomon(), typeCtx); + solomonState->Configuration->Init(gatewaysConfig->GetSolomon(), typeCtx); } TDataProviderInfo info; diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h index 4055df64a3e..7f4a88a1315 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h @@ -11,21 +11,21 @@ struct TSolomonState : public TThrRefBase { using TPtr = TIntrusivePtr<TSolomonState>; -public: - bool IsRtmrMode() const { - return SupportRtmrMode; - } - -public: - bool SupportRtmrMode = true; - +public: + bool IsRtmrMode() const { + return SupportRtmrMode; + } + +public: + bool SupportRtmrMode = true; + ISolomonGateway::TPtr Gateway; TTypeAnnotationContext* Types = nullptr; TSolomonConfiguration::TPtr Configuration = MakeIntrusive<TSolomonConfiguration>(); - THolder<IDqIntegration> DqIntegration; + THolder<IDqIntegration> DqIntegration; }; -TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode = true); +TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode = true); TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state); TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h index 831bbf98d12..c1f48a5b540 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h @@ -19,6 +19,6 @@ THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonSta THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state); THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state); -THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state); - +THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state); + } // namespace NYql diff --git a/ydb/library/yql/providers/solomon/ya.make b/ydb/library/yql/providers/solomon/ya.make index 54027e73832..579c088007d 100644 --- a/ydb/library/yql/providers/solomon/ya.make +++ b/ydb/library/yql/providers/solomon/ya.make @@ -1,7 +1,7 @@ RECURSE( - async_io + async_io expr_nodes gateway - proto + proto provider ) diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp index 05976b9dbfe..7ab55cfcfc9 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp @@ -192,7 +192,7 @@ private: RequestsDone = true; while(!Blocks.empty()) Blocks.pop(); - Callbacks->OnSourceError(InputIndex, res.GetIssues(), true); + Callbacks->OnSourceError(InputIndex, res.GetIssues(), true); } else { WakeUpTime = TMonotonic::Now() + Min(TDuration::Seconds(3), TDuration::MilliSeconds(0x30U * (1U << ++Retried))); ActorSystem->Schedule(WakeUpTime, new IEventHandle(SelfId(), TActorId(), new TEvPrivate::TEvRetryTime)); diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp index 0e648486b04..022df277833 100644 --- a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp +++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp @@ -2,13 +2,13 @@ #include "yql_ydb_read_actor.h" #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> - + namespace NYql::NDq { void RegisterYdbReadActorFactory(NYql::NDq::TDqSourceFactory& factory, ::NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { factory.Register<NYql::NYdb::TSource>("YdbSource", - [driver, credentialsFactory](NYql::NYdb::TSource&& settings, IDqSourceActorFactory::TArguments&& args) { - return CreateYdbReadActor(std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, driver, credentialsFactory); + [driver, credentialsFactory](NYql::NYdb::TSource&& settings, IDqSourceActorFactory::TArguments&& args) { + return CreateYdbReadActor(std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, driver, credentialsFactory); }); } diff --git a/ydb/library/yql/sql/v0/node.cpp b/ydb/library/yql/sql/v0/node.cpp index 4350b7dfb84..57f0c6a561f 100644 --- a/ydb/library/yql/sql/v0/node.cpp +++ b/ydb/library/yql/sql/v0/node.cpp @@ -1459,8 +1459,8 @@ TNodePtr ISource::BuildAggregation(const TString& label) { BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor), HoppingWindowSpec->Hop, HoppingWindowSpec->Interval, - HoppingWindowSpec->Delay, - Q("False")); + HoppingWindowSpec->Delay, + Q("False")); return Y("Aggregate", label, Q(keysTuple), Q(aggrArgs), Q(Y(Q(Y(BuildQuotedAtom(Pos, "hopping"), hoppingTraits))))); diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h index c916884ecb0..89517574862 100644 --- a/ydb/library/yql/sql/v1/context.h +++ b/ydb/library/yql/sql/v1/context.h @@ -246,7 +246,7 @@ namespace NSQLTranslationV1 { bool PositionalUnionAll = false; bool BogousStarInGroupByOverJoin = false; bool UnorderedSubqueries = true; - bool PragmaDataWatermarks = true; + bool PragmaDataWatermarks = true; bool WarnOnAnsiAliasShadowing = true; ui32 ResultRowsLimit = 0; ui64 ResultSizeLimit = 0; diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp index d0366be2b03..870bbd9cc7f 100644 --- a/ydb/library/yql/sql/v1/node.cpp +++ b/ydb/library/yql/sql/v1/node.cpp @@ -999,7 +999,7 @@ THoppingWindowSpecPtr THoppingWindowSpec::Clone() const { res->Hop = Hop->Clone(); res->Interval = Interval->Clone(); res->Delay = Delay->Clone(); - res->DataWatermarks = DataWatermarks; + res->DataWatermarks = DataWatermarks; return res; } @@ -1780,8 +1780,8 @@ TNodePtr ISource::BuildAggregation(const TString& label) { BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor), HoppingWindowSpec->Hop, HoppingWindowSpec->Interval, - HoppingWindowSpec->Delay, - HoppingWindowSpec->DataWatermarks ? Q("true") : Q("false")); + HoppingWindowSpec->Delay, + HoppingWindowSpec->DataWatermarks ? Q("true") : Q("false")); options = L(options, Q(Y(Q("hopping"), hoppingTraits))); } diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 768e5ffc518..de82b45a238 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -609,7 +609,7 @@ namespace NSQLTranslationV1 { TNodePtr Hop; TNodePtr Interval; TNodePtr Delay; - bool DataWatermarks; + bool DataWatermarks; TIntrusivePtr<THoppingWindowSpec> Clone() const; ~THoppingWindowSpec() {} diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index 7844bebcb6f..ab5a8647885 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -7407,7 +7407,7 @@ bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& nod if (!HoppingWindowSpec->Delay) { return false; } - HoppingWindowSpec->DataWatermarks = Ctx.PragmaDataWatermarks; + HoppingWindowSpec->DataWatermarks = Ctx.PragmaDataWatermarks; return true; } @@ -9574,22 +9574,22 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } else if (normalizedPragma == "disableunorderedsubqueries") { Ctx.UnorderedSubqueries = false; Ctx.IncrementMonCounter("sql_pragma", "DisableUnorderedSubqueries"); - } else if (normalizedPragma == "datawatermarks") { - if (values.size() != 1 || !values[0].GetLiteral() - || ! (*values[0].GetLiteral() == "enable" || *values[0].GetLiteral() == "disable")) - { - Error() << "Expected `enable|disable' argument for: " << pragma; - Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); - return {}; - } - - if (*values[0].GetLiteral() == "enable") { - Ctx.PragmaDataWatermarks = true; - } else if (*values[0].GetLiteral() == "disable") { - Ctx.PragmaDataWatermarks = false; - } - - Ctx.IncrementMonCounter("sql_pragma", "DataWatermarks"); + } else if (normalizedPragma == "datawatermarks") { + if (values.size() != 1 || !values[0].GetLiteral() + || ! (*values[0].GetLiteral() == "enable" || *values[0].GetLiteral() == "disable")) + { + Error() << "Expected `enable|disable' argument for: " << pragma; + Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); + return {}; + } + + if (*values[0].GetLiteral() == "enable") { + Ctx.PragmaDataWatermarks = true; + } else if (*values[0].GetLiteral() == "disable") { + Ctx.PragmaDataWatermarks = false; + } + + Ctx.IncrementMonCounter("sql_pragma", "DataWatermarks"); } else if (normalizedPragma == "flexibletypes") { Ctx.FlexibleTypes = true; Ctx.IncrementMonCounter("sql_pragma", "FlexibleTypes"); diff --git a/ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp b/ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp index d4da1071bca..a83941f90dd 100644 --- a/ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp +++ b/ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp @@ -1010,7 +1010,7 @@ void skipJSONField(ReadBuffer & buf, const StringRef & name_of_field) } else { - throw Exception("Unexpected symbol '" + std::string(buf.position(), 1) + "' for key '" + name_of_field.toString() + "'", ErrorCodes::INCORRECT_DATA); + throw Exception("Unexpected symbol '" + std::string(buf.position(), 1) + "' for key '" + name_of_field.toString() + "'", ErrorCodes::INCORRECT_DATA); } } diff --git a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 8157de7cf4b..1e6eac30556 100644 --- a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -61,7 +61,7 @@ namespace ErrorCodes extern const int UNKNOWN_TYPE; extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; extern const int THERE_IS_NO_COLUMN; - extern const int DUPLICATE_COLUMN; + extern const int DUPLICATE_COLUMN; extern const int BAD_ARGUMENTS; extern const int UNKNOWN_EXCEPTION; } @@ -529,9 +529,9 @@ void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr<arr for (const auto& column_name : table->ColumnNames()) { std::shared_ptr<arrow::ChunkedArray> arrow_column = table->GetColumnByName(column_name); - if (!arrow_column) { - throw Exception{ErrorCodes::DUPLICATE_COLUMN, "Column '{}' is duplicated.", column_name}; - } + if (!arrow_column) { + throw Exception{ErrorCodes::DUPLICATE_COLUMN, "Column '{}' is duplicated.", column_name}; + } name_to_column_ptr[column_name] = arrow_column; } diff --git a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp index 2e9776abf5c..927f612f27f 100644 --- a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp +++ b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes RawBLOBRowInputFormat::RawBLOBRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) : IRowInputFormat(header_, in_, std::move(params_)) { - if (header_.columns() > 1 || header_.columns() == 0) + if (header_.columns() > 1 || header_.columns() == 0) throw Exception(ErrorCodes::BAD_ARGUMENTS, "This input format is only suitable for tables with a single column of type String but the number of columns is {}", header_.columns()); diff --git a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index bf832960b90..85937935f11 100644 --- a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -52,7 +52,7 @@ static bool readName(ReadBuffer & buf, StringRef & ref, String & tmp) if (next_pos == buf.buffer().end()) { tmp.append(buf.position(), next_pos - buf.position()); - buf.position() = buf.buffer().end(); + buf.position() = buf.buffer().end(); buf.next(); continue; } diff --git a/ydb/library/yql/udfs/common/clickhouse/client/ya.make b/ydb/library/yql/udfs/common/clickhouse/client/ya.make index 106a6d0f85c..d9ef0a66458 100644 --- a/ydb/library/yql/udfs/common/clickhouse/client/ya.make +++ b/ydb/library/yql/udfs/common/clickhouse/client/ya.make @@ -419,7 +419,7 @@ IF (OS_LINUX AND CLANG AND NOT WITH_VALGRIND) ) CFLAGS ( - -DARCADIA_BUILD -DOS_LINUX -DUSE_ARROW=0 -DUSE_PARQUET=0 -DUSE_ORC=0 -DUSE_AVRO=0 -DUSE_UNWIND=0 -DDBMS_VERSION_MAJOR=21 -DDBMS_VERSION_MINOR=18 -DDBMS_VERSION_PATCH=0 + -DARCADIA_BUILD -DOS_LINUX -DUSE_ARROW=0 -DUSE_PARQUET=0 -DUSE_ORC=0 -DUSE_AVRO=0 -DUSE_UNWIND=0 -DDBMS_VERSION_MAJOR=21 -DDBMS_VERSION_MINOR=18 -DDBMS_VERSION_PATCH=0 -Wno-unused-parameter ) diff --git a/ydb/public/api/protos/yq.proto b/ydb/public/api/protos/yq.proto index ad9a13eae82..6aa7169a890 100644 --- a/ydb/public/api/protos/yq.proto +++ b/ydb/public/api/protos/yq.proto @@ -58,7 +58,7 @@ enum ExecuteMode { enum QueryAction { QUERY_ACTION_UNSPECIFIED = 0; PAUSE = 1; // Pause the query, with the possibility of its quick resumption - PAUSE_GRACEFULLY = 2; // Similar to PAUSE, only suspends the query allowing it to pause in checkpoint. Can work for a long time + PAUSE_GRACEFULLY = 2; // Similar to PAUSE, only suspends the query allowing it to pause in checkpoint. Can work for a long time ABORT = 3; // Stop the query ABORT_GRACEFULLY = 4; // Similar to ABORT, only stops the query in checkpoint RESUME = 5; // Resumes the execution of the query. Works only for PAUSE queries @@ -66,7 +66,7 @@ enum QueryAction { enum StateLoadMode { STATE_LOAD_MODE_UNSPECIFIED = 0; - EMPTY = 1; // Start the query with an empty state + EMPTY = 1; // Start the query with an empty state FROM_LAST_CHECKPOINT = 2; // Start the query with the state that is saved in the last checkpoint } @@ -89,8 +89,8 @@ message StreamingDisposition { } oneof disposition { - google.protobuf.Empty oldest = 1; // Start processing with the oldest offset - google.protobuf.Empty fresh = 2; // Start processing with the fresh offset + google.protobuf.Empty oldest = 1; // Start processing with the oldest offset + google.protobuf.Empty fresh = 2; // Start processing with the fresh offset FromTime from_time = 3; // Start processing with offset from the specified time TimeAgo time_ago = 4; // Start processing with offset some time ago FromLastCheckpoint from_last_checkpoint = 5; // Start processing with offset which corresponds to the last checkpoint @@ -418,12 +418,12 @@ message DataStreams { bool secure = 5; } -message Monitoring { - string project = 1 [(Ydb.length).le = 200]; - string cluster = 2 [(Ydb.length).le = 200]; - IamAuth auth = 3; -} - +message Monitoring { + string project = 1 [(Ydb.length).le = 200]; + string cluster = 2 [(Ydb.length).le = 200]; + IamAuth auth = 3; +} + message YdbDatabase { string database_id = 1 [(Ydb.length).le = 1024]; IamAuth auth = 2; @@ -466,7 +466,7 @@ message ConnectionSetting { ClickHouseCluster clickhouse_cluster = 2; DataStreams data_streams = 3; ObjectStorageConnection object_storage = 4; - Monitoring monitoring = 5; + Monitoring monitoring = 5; } } diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index 3927916c0ac..c067928c785 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -12,8 +12,8 @@ #include <contrib/libs/apache/arrow/cpp/src/arrow/csv/api.h> // for WriteCSV() #include <contrib/libs/apache/arrow/cpp/src/arrow/io/api.h> -#include <functional> - +#include <functional> + using namespace NKikimr; namespace NYdb { @@ -47,8 +47,8 @@ public: const TVector<NKikimrKqp::TKqpSetting>& kqpSettings = {}, TAutoPtr<TLogBackend> logBackend = {}, bool enableYq = false, - TAppPrepare::TFnReg udfFrFactory = nullptr, - std::function<void(TServerSettings& settings)> builder = nullptr) + TAppPrepare::TFnReg udfFrFactory = nullptr, + std::function<void(TServerSettings& settings)> builder = nullptr) { ui16 port = PortManager.GetPort(2134); ui16 grpc = PortManager.GetPort(2135); @@ -69,7 +69,7 @@ public: ServerSettings->AddStoragePoolType("hdd2"); } ServerSettings->SetAppConfig(appConfig); - ServerSettings->AuthConfig = appConfig.GetAuthConfig(); + ServerSettings->AuthConfig = appConfig.GetAuthConfig(); ServerSettings->FeatureFlags = appConfig.GetFeatureFlags(); ServerSettings->SetKqpSettings(kqpSettings); ServerSettings->SetEnableAsyncIndexes(true); @@ -87,9 +87,9 @@ public: if (udfFrFactory) { ServerSettings->SetFrFactory(udfFrFactory); } - if (builder) { - builder(*ServerSettings);; - } + if (builder) { + builder(*ServerSettings);; + } Server_.Reset(new TServer(*ServerSettings)); Tenants_.Reset(new Tests::TTenants(Server_)); |