aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authord-mokhnatkin <d-mokhnatkin@yandex-team.ru>2022-02-10 16:50:43 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:50:43 +0300
commitd94e393f3d35cf42ec17be6dc35149893a0b49cf (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8
parent4a2fe1d5dc0a49394dec49309de618073ccd14b7 (diff)
downloadydb-d94e393f3d35cf42ec17be6dc35149893a0b49cf.tar.gz
Restoring authorship annotation for <d-mokhnatkin@yandex-team.ru>. Commit 2 of 2.
-rw-r--r--library/cpp/actors/http/http_proxy.cpp22
-rw-r--r--library/cpp/streams/lz/lz.cpp14
-rw-r--r--library/cpp/streams/lz/lz_ut.cpp18
-rw-r--r--library/cpp/streams/lz/ut/ya.make2
-rw-r--r--library/cpp/streams/lz/ut/yq_609.databin722 -> 721 bytes
-rw-r--r--ydb/core/driver_lib/run/factories.h4
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp4
-rw-r--r--ydb/core/grpc_services/rpc_yq.cpp54
-rw-r--r--ydb/core/testlib/test_client.cpp10
-rw-r--r--ydb/core/yq/libs/actors/clusters_from_connections.cpp4
-rw-r--r--ydb/core/yq/libs/actors/run_actor.cpp26
-rw-r--r--ydb/core/yq/libs/actors/system_clusters.cpp24
-rw-r--r--ydb/core/yq/libs/audit/events/events.cpp2
-rw-r--r--ydb/core/yq/libs/audit/events/events.h360
-rw-r--r--ydb/core/yq/libs/audit/events/ya.make26
-rw-r--r--ydb/core/yq/libs/audit/mock/ya.make22
-rw-r--r--ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp142
-rw-r--r--ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h24
-rw-r--r--ydb/core/yq/libs/audit/ya.make26
-rw-r--r--ydb/core/yq/libs/audit/yq_audit_service.cpp26
-rw-r--r--ydb/core/yq/libs/audit/yq_audit_service.h24
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp16
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/storage_proxy.h6
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/storage_service.cpp6
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/storage_service.h6
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp14
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h4
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp22
-rw-r--r--ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h6
-rw-r--r--ydb/core/yq/libs/control_plane_storage/control_plane_storage.h8
-rw-r--r--ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp222
-rw-r--r--ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp10
-rw-r--r--ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h14
-rw-r--r--ydb/core/yq/libs/events/event_subspace.h2
-rw-r--r--ydb/core/yq/libs/init/init.cpp8
-rw-r--r--ydb/core/yq/libs/init/init.h4
-rw-r--r--ydb/core/yq/libs/shared_resources/db_pool.cpp18
-rw-r--r--ydb/core/yq/libs/shared_resources/db_pool.h8
-rw-r--r--ydb/core/yq/libs/shared_resources/shared_resources.cpp6
-rw-r--r--ydb/core/yq/libs/shared_resources/shared_resources.h4
-rw-r--r--ydb/core/yq/libs/ya.make2
-rw-r--r--ydb/core/yq/libs/ydb/ydb.cpp18
-rw-r--r--ydb/core/yq/libs/ydb/ydb.h4
-rw-r--r--ydb/library/security/ya.make2
-rw-r--r--ydb/library/security/ydb_credentials_provider_factory.cpp26
-rw-r--r--ydb/library/security/ydb_credentials_provider_factory.h40
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_extr_members.cpp10
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_flow2.cpp82
-rw-r--r--ydb/library/yql/core/expr_nodes/yql_expr_nodes.json76
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_core.cpp20
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_list.cpp414
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_list.h2
-rw-r--r--ydb/library/yql/core/yql_expr_type_annotation.cpp12
-rw-r--r--ydb/library/yql/core/yql_expr_type_annotation.h2
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp10
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h2
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h110
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp14
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h30
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h26
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h26
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp14
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp100
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h76
-rw-r--r--ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp172
-rw-r--r--ydb/library/yql/dq/actors/compute/ut/ya.make32
-rw-r--r--ydb/library/yql/dq/actors/compute/ya.make12
-rw-r--r--ydb/library/yql/dq/actors/protos/dq_stats.proto14
-rw-r--r--ydb/library/yql/dq/runtime/dq_output.h26
-rw-r--r--ydb/library/yql/dq/runtime/dq_output_channel.h2
-rw-r--r--ydb/library/yql/dq/runtime/dq_sink.cpp52
-rw-r--r--ydb/library/yql/dq/runtime/dq_sink.h16
-rw-r--r--ydb/library/yql/dq/runtime/dq_tasks_runner.cpp18
-rw-r--r--ydb/library/yql/dq/runtime/dq_tasks_runner.h14
-rw-r--r--ydb/library/yql/dq/type_ann/dq_type_ann.cpp8
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp4
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp1060
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h20
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp600
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp738
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/ya.make4
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ya.make4
-rw-r--r--ydb/library/yql/minikql/mkql_alloc.h2
-rw-r--r--ydb/library/yql/minikql/mkql_node.h4
-rw-r--r--ydb/library/yql/minikql/mkql_program_builder.cpp166
-rw-r--r--ydb/library/yql/minikql/mkql_program_builder.h20
-rw-r--r--ydb/library/yql/minikql/ut/ya.make2
-rw-r--r--ydb/library/yql/minikql/watermark_tracker.cpp78
-rw-r--r--ydb/library/yql/minikql/watermark_tracker.h50
-rw-r--r--ydb/library/yql/minikql/watermark_tracker_ut.cpp52
-rw-r--r--ydb/library/yql/minikql/ya.make4
-rw-r--r--ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp6
-rw-r--r--ydb/library/yql/providers/common/mkql/parser.cpp280
-rw-r--r--ydb/library/yql/providers/common/mkql/parser.h38
-rw-r--r--ydb/library/yql/providers/common/mkql/ya.make8
-rw-r--r--ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp104
-rw-r--r--ydb/library/yql/providers/common/proto/gateways_config.proto26
-rw-r--r--ydb/library/yql/providers/common/provider/yql_provider.cpp20
-rw-r--r--ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp270
-rw-r--r--ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h420
-rw-r--r--ydb/library/yql/providers/common/ut_helpers/ya.make30
-rw-r--r--ydb/library/yql/providers/dq/actors/task_controller.cpp86
-rw-r--r--ydb/library/yql/providers/dq/common/yql_dq_settings.cpp2
-rw-r--r--ydb/library/yql/providers/dq/common/yql_dq_settings.h4
-rw-r--r--ydb/library/yql/providers/dq/opt/logical_optimize.cpp1376
-rw-r--r--ydb/library/yql/providers/dq/opt/logical_optimize.h2
-rw-r--r--ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp6
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp2
-rw-r--r--ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp564
-rw-r--r--ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h42
-rw-r--r--ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp680
-rw-r--r--ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h38
-rw-r--r--ydb/library/yql/providers/pq/async_io/probes.cpp6
-rw-r--r--ydb/library/yql/providers/pq/async_io/probes.h34
-rw-r--r--ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp258
-rw-r--r--ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp206
-rw-r--r--ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp324
-rw-r--r--ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h76
-rw-r--r--ydb/library/yql/providers/pq/async_io/ut/ya.make42
-rw-r--r--ydb/library/yql/providers/pq/async_io/ya.make40
-rw-r--r--ydb/library/yql/providers/pq/cm_client/lib/client.cpp2
-rw-r--r--ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp6
-rw-r--r--ydb/library/yql/providers/pq/proto/dq_io.proto64
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp8
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp14
-rw-r--r--ydb/library/yql/providers/pq/ya.make2
-rw-r--r--ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp2
-rw-r--r--ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp2
-rw-r--r--ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp8
-rw-r--r--ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp886
-rw-r--r--ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h46
-rw-r--r--ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp296
-rw-r--r--ydb/library/yql/providers/solomon/async_io/metrics_encoder.h58
-rw-r--r--ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp364
-rw-r--r--ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp246
-rw-r--r--ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h54
-rw-r--r--ydb/library/yql/providers/solomon/async_io/ut/ya.make40
-rw-r--r--ydb/library/yql/providers/solomon/async_io/ya.make42
-rw-r--r--ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json42
-rw-r--r--ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto76
-rw-r--r--ydb/library/yql/providers/solomon/proto/ya.make26
-rw-r--r--ydb/library/yql/providers/solomon/provider/ya.make20
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_config.h12
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp28
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp212
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp68
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp278
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h24
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp306
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp10
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h20
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h4
-rw-r--r--ydb/library/yql/providers/solomon/ya.make4
-rw-r--r--ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp2
-rw-r--r--ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp6
-rw-r--r--ydb/library/yql/sql/v0/node.cpp4
-rw-r--r--ydb/library/yql/sql/v1/context.h2
-rw-r--r--ydb/library/yql/sql/v1/node.cpp6
-rw-r--r--ydb/library/yql/sql/v1/node.h2
-rw-r--r--ydb/library/yql/sql/v1/sql.cpp34
-rw-r--r--ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp2
-rw-r--r--ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp8
-rw-r--r--ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp2
-rw-r--r--ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp2
-rw-r--r--ydb/library/yql/udfs/common/clickhouse/client/ya.make2
-rw-r--r--ydb/public/api/protos/yq.proto22
-rw-r--r--ydb/services/ydb/ydb_common_ut.h16
167 files changed, 6940 insertions, 6940 deletions
diff --git a/library/cpp/actors/http/http_proxy.cpp b/library/cpp/actors/http/http_proxy.cpp
index 4b57a8453dc..36c6855d93f 100644
--- a/library/cpp/actors/http/http_proxy.cpp
+++ b/library/cpp/actors/http/http_proxy.cpp
@@ -1,4 +1,4 @@
-#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/events.h>
#include <library/cpp/monlib/metrics/metric_registry.h>
#include "http_proxy.h"
@@ -43,19 +43,19 @@ protected:
HFunc(TEvHttpProxy::TEvHttpConnectionClosed, Handle);
HFunc(TEvHttpProxy::TEvResolveHostRequest, Handle);
HFunc(TEvHttpProxy::TEvReportSensors, Handle);
- HFunc(NActors::TEvents::TEvPoison, Handle);
+ HFunc(NActors::TEvents::TEvPoison, Handle);
}
}
- void PassAway() override {
- Send(Poller, new NActors::TEvents::TEvPoisonPill());
+ void PassAway() override {
+ Send(Poller, new NActors::TEvents::TEvPoisonPill());
for (const NActors::TActorId& connection : Connections) {
- Send(connection, new NActors::TEvents::TEvPoisonPill());
+ Send(connection, new NActors::TEvents::TEvPoisonPill());
}
for (const NActors::TActorId& acceptor : Acceptors) {
- Send(acceptor, new NActors::TEvents::TEvPoisonPill());
+ Send(acceptor, new NActors::TEvents::TEvPoisonPill());
}
- NActors::TActorBootstrapped<THttpProxy>::PassAway();
+ NActors::TActorBootstrapped<THttpProxy>::PassAway();
}
void Handle(TEvHttpProxy::TEvHttpIncomingRequest::TPtr event, const NActors::TActorContext& ctx) {
@@ -200,10 +200,10 @@ protected:
NMonitoring::ExplicitHistogram({1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 30000, 60000}))->Record(sensors.Time.MilliSeconds());
}
- void Handle(NActors::TEvents::TEvPoison::TPtr, const NActors::TActorContext&) {
- PassAway();
- }
-
+ void Handle(NActors::TEvents::TEvPoison::TPtr, const NActors::TActorContext&) {
+ PassAway();
+ }
+
NActors::TActorId Poller;
TVector<TActorId> Acceptors;
diff --git a/library/cpp/streams/lz/lz.cpp b/library/cpp/streams/lz/lz.cpp
index de090a257c9..b65bb3ed965 100644
--- a/library/cpp/streams/lz/lz.cpp
+++ b/library/cpp/streams/lz/lz.cpp
@@ -3,7 +3,7 @@
#include <util/system/yassert.h>
#include <util/system/byteorder.h>
#include <util/memory/addstorage.h>
-#include <util/generic/buffer.h>
+#include <util/generic/buffer.h>
#include <util/generic/utility.h>
#include <util/generic/singleton.h>
#include <util/generic/yexception.h>
@@ -193,8 +193,8 @@ public:
, Version_(CheckVer(Load<ui32>()))
, BlockSize_(Load<ui16>())
, OutBufSize_(TDecompressor::Hint(BlockSize_))
- , Tmp_(2 * OutBufSize_)
- , In_(Tmp_.Data())
+ , Tmp_(2 * OutBufSize_)
+ , In_(Tmp_.Data())
, Out_(In_ + OutBufSize_)
{
this->InitFromStream(Slave_);
@@ -237,9 +237,9 @@ public:
TMemoryInput header(tmp, sizeof(tmp));
const ui16 len = GLoad<ui16>(&header);
- if (len > Tmp_.Capacity()) {
- ythrow TDecompressorError() << "invalid len inside block header";
- }
+ if (len > Tmp_.Capacity()) {
+ ythrow TDecompressorError() << "invalid len inside block header";
+ }
const ui8 compressed = GLoad<ui8>(&header);
if (compressed > 1) {
@@ -271,7 +271,7 @@ protected:
const ui32 Version_;
const ui16 BlockSize_;
const size_t OutBufSize_;
- TBuffer Tmp_;
+ TBuffer Tmp_;
char* In_;
char* Out_;
};
diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp
index 5949f397c1b..6876f070fc0 100644
--- a/library/cpp/streams/lz/lz_ut.cpp
+++ b/library/cpp/streams/lz/lz_ut.cpp
@@ -275,13 +275,13 @@ Y_UNIT_TEST_SUITE(TLzTest) {
UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789");
}
-
- Y_UNIT_TEST(TestYQ609) {
- auto data = NResource::Find("/yq_609.data");
-
- TMemoryInput input(data.Data(), data.Size());
-
- TLz4Decompress d(&input);
- UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError);
- }
+
+ Y_UNIT_TEST(TestYQ609) {
+ auto data = NResource::Find("/yq_609.data");
+
+ TMemoryInput input(data.Data(), data.Size());
+
+ TLz4Decompress d(&input);
+ UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError);
+ }
}
diff --git a/library/cpp/streams/lz/ut/ya.make b/library/cpp/streams/lz/ut/ya.make
index ca35b53f101..18288c8ac91 100644
--- a/library/cpp/streams/lz/ut/ya.make
+++ b/library/cpp/streams/lz/ut/ya.make
@@ -8,7 +8,7 @@ OWNER(
RESOURCE(
random.data /random.data
request.data /request.data
- yq_609.data /yq_609.data
+ yq_609.data /yq_609.data
)
SRCS(
diff --git a/library/cpp/streams/lz/ut/yq_609.data b/library/cpp/streams/lz/ut/yq_609.data
index 144f4b4cc70..9694b0c97f1 100644
--- a/library/cpp/streams/lz/ut/yq_609.data
+++ b/library/cpp/streams/lz/ut/yq_609.data
Binary files differ
diff --git a/ydb/core/driver_lib/run/factories.h b/ydb/core/driver_lib/run/factories.h
index 92ad3f8b5e2..41a45b44e08 100644
--- a/ydb/core/driver_lib/run/factories.h
+++ b/ydb/core/driver_lib/run/factories.h
@@ -19,7 +19,7 @@
#include <library/cpp/actors/core/actorsystem.h>
#include <ydb/library/security/ydb_credentials_provider_factory.h>
-
+
#include <functional>
#include <unordered_map>
#include <unordered_set>
@@ -46,7 +46,7 @@ struct TModuleFactories {
IActor*(*FolderServiceFactory)(const NKikimrProto::NFolderService::TFolderServiceConfig&);
std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)> YqAuditServiceFactory;
- NKikimr::TYdbCredentialsProviderFactory YdbCredentialProviderFactory;
+ NKikimr::TYdbCredentialsProviderFactory YdbCredentialProviderFactory;
// Factory for grpc services
TGrpcServiceFactory GrpcServiceFactory;
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index 905a0e7027a..819c1478d19 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -182,8 +182,8 @@
#include <util/digest/city.h>
#include <util/generic/algorithm.h>
-#include <util/generic/size_literals.h>
-
+#include <util/generic/size_literals.h>
+
#include <util/system/hostname.h>
namespace NKikimr {
diff --git a/ydb/core/grpc_services/rpc_yq.cpp b/ydb/core/grpc_services/rpc_yq.cpp
index eca0203857b..6fe5f868d6b 100644
--- a/ydb/core/grpc_services/rpc_yq.cpp
+++ b/ydb/core/grpc_services/rpc_yq.cpp
@@ -35,13 +35,13 @@ public:
using TBase::GetProtoRequest;
protected:
- TString Token;
- TString FolderId;
- TString User;
- TString PeerName;
- TString UserAgent;
- TString RequestId;
-
+ TString Token;
+ TString FolderId;
+ TString User;
+ TString PeerName;
+ TString UserAgent;
+ TString RequestId;
+
public:
TYandexQueryRequestRPC(IRequestOpCtx* request)
: TBase(request) {}
@@ -51,7 +51,7 @@ public:
auto request = dynamic_cast<RpcRequestType*>(requestCtx);
Y_VERIFY(request);
-
+
auto proxyCtx = dynamic_cast<IRequestProxyCtx*>(requestCtx);
Y_VERIFY(proxyCtx);
@@ -78,13 +78,13 @@ public:
return;
}
- FolderId = path.back();
- if (!FolderId) {
+ FolderId = path.back();
+ if (!FolderId) {
ReplyWithStatus("Folder id is empty", StatusIds::BAD_REQUEST);
return;
}
- if (FolderId.length() > 1024) {
+ if (FolderId.length() > 1024) {
ReplyWithStatus("Folder id length greater than 1024 characters: " + FolderId, StatusIds::BAD_REQUEST);
return;
}
@@ -93,7 +93,7 @@ public:
TVector<TString> permissions;
if (internalToken) {
NACLib::TUserToken userToken(internalToken);
- User = userToken.GetUserSID();
+ User = userToken.GetUserSID();
for (const auto& sid: request->Sids) {
if (userToken.IsExist(sid)) {
permissions.push_back(sid);
@@ -101,7 +101,7 @@ public:
}
}
- if (!User) {
+ if (!User) {
ReplyWithStatus("Authorization error. Permission denied", StatusIds::UNAUTHORIZED);
return;
}
@@ -142,20 +142,20 @@ protected:
req.SendResult(response.Result, StatusIds::SUCCESS);
}
- NYq::TEvAuditService::TExtraInfo extraInfo{
- .Token = Token,
- .FolderId = FolderId,
- .User = User,
- .PeerName = PeerName,
- .UserAgent = UserAgent,
- .RequestId = RequestId,
- };
-
- Send(NYq::YqAuditServiceActorId(), NYq::TEvAuditService::MakeAuditEvent(
- std::move(extraInfo),
- *GetProtoRequest(),
- response.Issues,
- response.AuditDetails));
+ NYq::TEvAuditService::TExtraInfo extraInfo{
+ .Token = Token,
+ .FolderId = FolderId,
+ .User = User,
+ .PeerName = PeerName,
+ .UserAgent = UserAgent,
+ .RequestId = RequestId,
+ };
+
+ Send(NYq::YqAuditServiceActorId(), NYq::TEvAuditService::MakeAuditEvent(
+ std::move(extraInfo),
+ *GetProtoRequest(),
+ response.Issues,
+ response.AuditDetails));
}
void Handle(typename EvResponseType::TPtr& ev) {
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index 52b4ca916a6..d4907c26f1e 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -723,7 +723,7 @@ namespace Tests {
auto& controlPlaneProxyConfig = *protoConfig.MutableControlPlaneProxy();
controlPlaneProxyConfig.SetEnabled(true);
}
-
+
{
auto& testConnectionConfig = *protoConfig.MutableTestConnection();
testConnectionConfig.SetEnabled(true);
@@ -752,7 +752,7 @@ namespace Tests {
commonConfig.SetYdbMvpCloudEndpoint(ydbMvpEndpoint);
commonConfig.SetIdsPrefix("ut");
}
-
+
{
auto& privateApiConfig = *protoConfig.MutablePrivateApi();
privateApiConfig.SetEnabled(true);
@@ -800,7 +800,7 @@ namespace Tests {
Runtime->RegisterService(serviceActorId, actorId, nodeIdx);
};
- const auto ydbCredFactory = NKikimr::CreateYdbCredentialsProviderFactory;
+ const auto ydbCredFactory = NKikimr::CreateYdbCredentialsProviderFactory;
auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
auto yqSharedResources = NYq::CreateYqSharedResources(protoConfig, ydbCredFactory, counters);
NYq::Init(
@@ -811,8 +811,8 @@ namespace Tests {
"TestTenant",
nullptr, // MakeIntrusive<NPq::NConfigurationManager::TConnections>(),
yqSharedResources,
- NKikimr::NFolderService::CreateMockFolderServiceActor,
- NYq::CreateMockYqAuditServiceActor,
+ NKikimr::NFolderService::CreateMockFolderServiceActor,
+ NYq::CreateMockYqAuditServiceActor,
ydbCredFactory,
/*IcPort = */0
);
diff --git a/ydb/core/yq/libs/actors/clusters_from_connections.cpp b/ydb/core/yq/libs/actors/clusters_from_connections.cpp
index 50b22babc74..93e04718927 100644
--- a/ydb/core/yq/libs/actors/clusters_from_connections.cpp
+++ b/ydb/core/yq/libs/actors/clusters_from_connections.cpp
@@ -111,12 +111,12 @@ void AddClustersFromConnections(const THashMap<TString, YandexQuery::Connection>
if (solomonEndpoint.empty()) {
if (connectionName.StartsWith("pre")) {
solomonEndpoint = "monitoring.api.cloud-preprod.yandex.net";
- clusterCfg->SetUseSsl(true);
+ clusterCfg->SetUseSsl(true);
} else if (connectionName.StartsWith("so")) {
solomonEndpoint = "solomon.yandex.net";
} else {
solomonEndpoint = "monitoring.api.cloud.yandex.net";
- clusterCfg->SetUseSsl(true);
+ clusterCfg->SetUseSsl(true);
}
}
diff --git a/ydb/core/yq/libs/actors/run_actor.cpp b/ydb/core/yq/libs/actors/run_actor.cpp
index 14eeaf2e611..5549f8f254e 100644
--- a/ydb/core/yq/libs/actors/run_actor.cpp
+++ b/ydb/core/yq/libs/actors/run_actor.cpp
@@ -856,7 +856,7 @@ private:
TDqConfiguration::TPtr dqConfiguration = MakeIntrusive<TDqConfiguration>();
dqConfiguration->Dispatch(dqGraphParams.GetSettings());
dqConfiguration->FreezeDefaults();
- dqConfiguration->FallbackPolicy = "never";
+ dqConfiguration->FallbackPolicy = "never";
ExecuterId = NActors::TActivationContext::Register(NYql::NDq::MakeDqExecuter(MakeYqlNodesManagerId(), SelfId(), Params.QueryId, "", dqConfiguration, ServiceCounters.Counters, TInstant::Now(), EnableCheckpointCoordinator));
@@ -937,12 +937,12 @@ private:
attr->SetName("_OneGraphPerQuery");
attr->SetValue("1");
}
-
- if (Params.QueryType == YandexQuery::QueryContent_QueryType_ANALYTICS) {
- attr = dqSettings.Add();
- attr->SetName("AnalyticsHopping");
- attr->SetValue("1");
- }
+
+ if (Params.QueryType == YandexQuery::QueryContent_QueryType_ANALYTICS) {
+ attr = dqSettings.Add();
+ attr->SetName("AnalyticsHopping");
+ attr->SetValue("1");
+ }
}
void AddClustersFromConfig(NYql::TGatewaysConfig& gatewaysConfig, THashMap<TString, TString>& clusters) const {
@@ -1163,12 +1163,12 @@ private:
dataProvidersInit.push_back(GetPqDataProviderInitializer(pqGateway, false, dbResolver));
}
- {
- auto solomonConfig = gatewaysConfig.GetSolomon();
- auto solomonGateway = NYql::CreateSolomonGateway(solomonConfig);
- dataProvidersInit.push_back(GetSolomonDataProviderInitializer(solomonGateway, false));
- }
-
+ {
+ auto solomonConfig = gatewaysConfig.GetSolomon();
+ auto solomonGateway = NYql::CreateSolomonGateway(solomonConfig);
+ dataProvidersInit.push_back(GetSolomonDataProviderInitializer(solomonGateway, false));
+ }
+
TProgramFactory progFactory(false, Params.FunctionRegistry, Params.NextUniqueId, dataProvidersInit, "yq");
progFactory.SetModules(Params.ModuleResolver);
progFactory.SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(Params.FunctionRegistry, nullptr));
diff --git a/ydb/core/yq/libs/actors/system_clusters.cpp b/ydb/core/yq/libs/actors/system_clusters.cpp
index bfa8eb99020..2968ae64872 100644
--- a/ydb/core/yq/libs/actors/system_clusters.cpp
+++ b/ydb/core/yq/libs/actors/system_clusters.cpp
@@ -19,16 +19,16 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin
}
{
const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping();
- clusterCfg->SetName("logbroker_iam_no_sa");
- clusterCfg->SetEndpoint("logbroker.yandex.net:2135");
- clusterCfg->SetConfigManagerEndpoint("cm.logbroker.yandex.net:1111");
- clusterCfg->SetAddBearerToToken(true);
- clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE);
- clusterCfg->SetToken(authToken);
- clusters.emplace(clusterCfg->GetName(), PqProviderName);
- }
- {
- const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping();
+ clusterCfg->SetName("logbroker_iam_no_sa");
+ clusterCfg->SetEndpoint("logbroker.yandex.net:2135");
+ clusterCfg->SetConfigManagerEndpoint("cm.logbroker.yandex.net:1111");
+ clusterCfg->SetAddBearerToToken(true);
+ clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE);
+ clusterCfg->SetToken(authToken);
+ clusters.emplace(clusterCfg->GetName(), PqProviderName);
+ }
+ {
+ const auto clusterCfg = gatewaysConfig.MutablePq()->AddClusterMapping();
clusterCfg->SetName("lbkx");
clusterCfg->SetEndpoint("lbkx.logbroker.yandex.net:2135");
clusterCfg->SetConfigManagerEndpoint("cm.lbkx.logbroker.yandex.net:1111");
@@ -89,7 +89,7 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin
clusterCfg->SetEndpoint("lb.cc8035oc71oh9um52mv3.ydb.mdb.cloud-preprod.yandex.net:2135");
clusterCfg->SetConfigManagerEndpoint("cm.global.logbroker.cloud-preprod.yandex.net:1111");
clusterCfg->SetDatabase("/pre-prod_global/aoeb66ftj1tbt1b2eimn/cc8035oc71oh9um52mv3");
- clusterCfg->SetUseSsl(true);
+ clusterCfg->SetUseSsl(true);
clusterCfg->SetServiceAccountId("bfbf4nc8c849ej11k3aq"); // audit-trails-reader-sa
clusterCfg->SetServiceAccountIdSignature("SyFk1SpKdQa6L8IJxnwvgC57yMQ=");
clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE);
@@ -101,7 +101,7 @@ void AddSystemClusters(TGatewaysConfig& gatewaysConfig, THashMap<TString, TStrin
clusterCfg->SetEndpoint("lb.etn03iai600jur7pipla.ydb.mdb.yandexcloud.net:2135");
clusterCfg->SetConfigManagerEndpoint("cm.global.logbroker.cloud.yandex.net:1111");
clusterCfg->SetDatabase("/global/b1gvcqr959dbmi1jltep/etn03iai600jur7pipla");
- clusterCfg->SetUseSsl(true);
+ clusterCfg->SetUseSsl(true);
clusterCfg->SetServiceAccountId("aje0ir9755dd4ia9mpu5");
clusterCfg->SetServiceAccountIdSignature("xxx");
clusterCfg->SetClusterType(TPqClusterConfig::CT_PERS_QUEUE);
diff --git a/ydb/core/yq/libs/audit/events/events.cpp b/ydb/core/yq/libs/audit/events/events.cpp
index a4930159cf2..6c3d2603e7e 100644
--- a/ydb/core/yq/libs/audit/events/events.cpp
+++ b/ydb/core/yq/libs/audit/events/events.cpp
@@ -1 +1 @@
-#include "events.h"
+#include "events.h"
diff --git a/ydb/core/yq/libs/audit/events/events.h b/ydb/core/yq/libs/audit/events/events.h
index 75cf23da197..15026dc082a 100644
--- a/ydb/core/yq/libs/audit/events/events.h
+++ b/ydb/core/yq/libs/audit/events/events.h
@@ -1,186 +1,186 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/public/issue/yql_issue.h>
-
+
#include <ydb/public/api/protos/yq.pb.h>
#include <ydb/core/yq/libs/control_plane_storage/events/events.h>
#include <ydb/core/yq/libs/checkpointing_common/defs.h>
-
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/event_pb.h>
-#include <library/cpp/actors/interconnect/events_local.h>
-
-#include <optional>
-
-namespace NYq {
-
-struct TEvAuditService {
- struct TExtraInfo {
- TString Token;
- TString CloudId;
- TString FolderId;
- TString User;
-
- TString PeerName;
- TString UserAgent;
- TString RequestId;
- };
-
- // Event ids.
- enum EEv : ui32 {
+
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/event_pb.h>
+#include <library/cpp/actors/interconnect/events_local.h>
+
+#include <optional>
+
+namespace NYq {
+
+struct TEvAuditService {
+ struct TExtraInfo {
+ TString Token;
+ TString CloudId;
+ TString FolderId;
+ TString User;
+
+ TString PeerName;
+ TString UserAgent;
+ TString RequestId;
+ };
+
+ // Event ids.
+ enum EEv : ui32 {
EvCreateBindingReport = YqEventSubspaceBegin(NYq::TYqEventSubspace::AuditService),
- EvModifyBindingReport,
- EvDeleteBindingReport,
- EvCreateConnectionReport,
- EvModifyConnectionReport,
- EvDeleteConnectionReport,
- EvCreateQueryReport,
- EvControlQueryReport,
- EvModifyQueryReport,
- EvDeleteQueryReport,
- EvEnd,
- };
-
+ EvModifyBindingReport,
+ EvDeleteBindingReport,
+ EvCreateConnectionReport,
+ EvModifyConnectionReport,
+ EvDeleteConnectionReport,
+ EvCreateQueryReport,
+ EvControlQueryReport,
+ EvModifyQueryReport,
+ EvDeleteQueryReport,
+ EvEnd,
+ };
+
static_assert(EvEnd <= YqEventSubspaceEnd(NYq::TYqEventSubspace::AuditService), "All events must be in their subspace");
-
-private:
- template <class TRequest, class TAuditDetailsObj>
- struct TAuditReportBase {
- explicit TAuditReportBase(
- TExtraInfo&& extraInfo,
- const TRequest& request,
- const NYql::TIssues& issues,
- const TAuditDetails<TAuditDetailsObj>& details,
- std::optional<TString> eventId = std::nullopt)
- : ExtraInfo(std::move(extraInfo))
- , Request(request)
- , Issues(issues)
- , Details(details)
- , EventId(eventId)
- {
- }
-
- TExtraInfo ExtraInfo;
- TRequest Request;
- NYql::TIssues Issues;
- TAuditDetails<TAuditDetailsObj> Details;
- std::optional<TString> EventId;
- };
-
-public:
- template <class TRequest, class TAuditDetailsObj>
- struct TAuditReport;
-
- // CreateBinding
- using CreateBindingAuditReport = TAuditReport<YandexQuery::CreateBindingRequest, YandexQuery::Binding>;
-
- template<>
- struct TAuditReport<YandexQuery::CreateBindingRequest, YandexQuery::Binding>
- : NActors::TEventLocal<CreateBindingAuditReport, EvCreateBindingReport>
- , TAuditReportBase<YandexQuery::CreateBindingRequest, YandexQuery::Binding> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // ModifyBinding
- using ModifyBindingAuditReport = TAuditReport<YandexQuery::ModifyBindingRequest, YandexQuery::Binding>;
-
- template<>
- struct TAuditReport<YandexQuery::ModifyBindingRequest, YandexQuery::Binding>
- : NActors::TEventLocal<ModifyBindingAuditReport, EvModifyBindingReport>
- , TAuditReportBase<YandexQuery::ModifyBindingRequest, YandexQuery::Binding> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // DeleteBinding
- using DeleteBindingAuditReport = TAuditReport<YandexQuery::DeleteBindingRequest, YandexQuery::Binding>;
-
- template<>
- struct TAuditReport<YandexQuery::DeleteBindingRequest, YandexQuery::Binding>
- : NActors::TEventLocal<DeleteBindingAuditReport, EvDeleteBindingReport>
- , TAuditReportBase<YandexQuery::DeleteBindingRequest, YandexQuery::Binding> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // CreateConnection
- using CreateConnectionAuditReport = TAuditReport<YandexQuery::CreateConnectionRequest, YandexQuery::Connection>;
-
- template<>
- struct TAuditReport<YandexQuery::CreateConnectionRequest, YandexQuery::Connection>
- : NActors::TEventLocal<CreateConnectionAuditReport, EvCreateConnectionReport>
- , TAuditReportBase<YandexQuery::CreateConnectionRequest, YandexQuery::Connection> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // ModifyConnection
- using ModifyConnectionAuditReport = TAuditReport<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection>;
-
- template<>
- struct TAuditReport<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection>
- : NActors::TEventLocal<ModifyConnectionAuditReport, EvModifyConnectionReport>
- , TAuditReportBase<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // DeleteConnection
- using DeleteConnectionAuditReport = TAuditReport<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection>;
-
- template<>
- struct TAuditReport<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection>
- : NActors::TEventLocal<DeleteConnectionAuditReport, EvDeleteConnectionReport>
- , TAuditReportBase<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // CreateQuery
- using CreateQueryAuditReport = TAuditReport<YandexQuery::CreateQueryRequest, YandexQuery::Query>;
-
- template<>
- struct TAuditReport<YandexQuery::CreateQueryRequest, YandexQuery::Query>
- : NActors::TEventLocal<CreateQueryAuditReport, EvCreateQueryReport>
- , TAuditReportBase<YandexQuery::CreateQueryRequest, YandexQuery::Query> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // ControlQuery
- using ControlQueryAuditReport = TAuditReport<YandexQuery::ControlQueryRequest, YandexQuery::Query>;
-
- template<>
- struct TAuditReport<YandexQuery::ControlQueryRequest, YandexQuery::Query>
- : NActors::TEventLocal<ControlQueryAuditReport, EvControlQueryReport>
- , TAuditReportBase<YandexQuery::ControlQueryRequest, YandexQuery::Query> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // ModifyQuery
- using ModifyQueryAuditReport = TAuditReport<YandexQuery::ModifyQueryRequest, YandexQuery::Query>;
-
- template<>
- struct TAuditReport<YandexQuery::ModifyQueryRequest, YandexQuery::Query>
- : NActors::TEventLocal<ModifyQueryAuditReport, EvModifyQueryReport>
- , TAuditReportBase<YandexQuery::ModifyQueryRequest, YandexQuery::Query> {
- using TAuditReportBase::TAuditReportBase;
- };
-
- // DeleteQuery
- using DeleteQueryAuditReport = TAuditReport<YandexQuery::DeleteQueryRequest, YandexQuery::Query>;
-
- template<>
- struct TAuditReport<YandexQuery::DeleteQueryRequest, YandexQuery::Query>
- : NActors::TEventLocal<DeleteQueryAuditReport, EvDeleteQueryReport>
- , TAuditReportBase<YandexQuery::DeleteQueryRequest, YandexQuery::Query> {
- using TAuditReportBase::TAuditReportBase;
- };
-
-
- template <class TRequest, class TAuditDetailsObj>
- static TAuditReport<TRequest, TAuditDetailsObj>* MakeAuditEvent(
- TExtraInfo&& extraInfo,
- const TRequest& request,
- const NYql::TIssues& issues,
- const TAuditDetails<TAuditDetailsObj>& details,
- std::optional<TString> eventId = std::nullopt) {
- return new TAuditReport<TRequest, TAuditDetailsObj>(std::move(extraInfo), request, issues, details, eventId);
- }
-};
-
-} // namespace NYq
+
+private:
+ template <class TRequest, class TAuditDetailsObj>
+ struct TAuditReportBase {
+ explicit TAuditReportBase(
+ TExtraInfo&& extraInfo,
+ const TRequest& request,
+ const NYql::TIssues& issues,
+ const TAuditDetails<TAuditDetailsObj>& details,
+ std::optional<TString> eventId = std::nullopt)
+ : ExtraInfo(std::move(extraInfo))
+ , Request(request)
+ , Issues(issues)
+ , Details(details)
+ , EventId(eventId)
+ {
+ }
+
+ TExtraInfo ExtraInfo;
+ TRequest Request;
+ NYql::TIssues Issues;
+ TAuditDetails<TAuditDetailsObj> Details;
+ std::optional<TString> EventId;
+ };
+
+public:
+ template <class TRequest, class TAuditDetailsObj>
+ struct TAuditReport;
+
+ // CreateBinding
+ using CreateBindingAuditReport = TAuditReport<YandexQuery::CreateBindingRequest, YandexQuery::Binding>;
+
+ template<>
+ struct TAuditReport<YandexQuery::CreateBindingRequest, YandexQuery::Binding>
+ : NActors::TEventLocal<CreateBindingAuditReport, EvCreateBindingReport>
+ , TAuditReportBase<YandexQuery::CreateBindingRequest, YandexQuery::Binding> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // ModifyBinding
+ using ModifyBindingAuditReport = TAuditReport<YandexQuery::ModifyBindingRequest, YandexQuery::Binding>;
+
+ template<>
+ struct TAuditReport<YandexQuery::ModifyBindingRequest, YandexQuery::Binding>
+ : NActors::TEventLocal<ModifyBindingAuditReport, EvModifyBindingReport>
+ , TAuditReportBase<YandexQuery::ModifyBindingRequest, YandexQuery::Binding> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // DeleteBinding
+ using DeleteBindingAuditReport = TAuditReport<YandexQuery::DeleteBindingRequest, YandexQuery::Binding>;
+
+ template<>
+ struct TAuditReport<YandexQuery::DeleteBindingRequest, YandexQuery::Binding>
+ : NActors::TEventLocal<DeleteBindingAuditReport, EvDeleteBindingReport>
+ , TAuditReportBase<YandexQuery::DeleteBindingRequest, YandexQuery::Binding> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // CreateConnection
+ using CreateConnectionAuditReport = TAuditReport<YandexQuery::CreateConnectionRequest, YandexQuery::Connection>;
+
+ template<>
+ struct TAuditReport<YandexQuery::CreateConnectionRequest, YandexQuery::Connection>
+ : NActors::TEventLocal<CreateConnectionAuditReport, EvCreateConnectionReport>
+ , TAuditReportBase<YandexQuery::CreateConnectionRequest, YandexQuery::Connection> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // ModifyConnection
+ using ModifyConnectionAuditReport = TAuditReport<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection>;
+
+ template<>
+ struct TAuditReport<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection>
+ : NActors::TEventLocal<ModifyConnectionAuditReport, EvModifyConnectionReport>
+ , TAuditReportBase<YandexQuery::ModifyConnectionRequest, YandexQuery::Connection> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // DeleteConnection
+ using DeleteConnectionAuditReport = TAuditReport<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection>;
+
+ template<>
+ struct TAuditReport<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection>
+ : NActors::TEventLocal<DeleteConnectionAuditReport, EvDeleteConnectionReport>
+ , TAuditReportBase<YandexQuery::DeleteConnectionRequest, YandexQuery::Connection> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // CreateQuery
+ using CreateQueryAuditReport = TAuditReport<YandexQuery::CreateQueryRequest, YandexQuery::Query>;
+
+ template<>
+ struct TAuditReport<YandexQuery::CreateQueryRequest, YandexQuery::Query>
+ : NActors::TEventLocal<CreateQueryAuditReport, EvCreateQueryReport>
+ , TAuditReportBase<YandexQuery::CreateQueryRequest, YandexQuery::Query> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // ControlQuery
+ using ControlQueryAuditReport = TAuditReport<YandexQuery::ControlQueryRequest, YandexQuery::Query>;
+
+ template<>
+ struct TAuditReport<YandexQuery::ControlQueryRequest, YandexQuery::Query>
+ : NActors::TEventLocal<ControlQueryAuditReport, EvControlQueryReport>
+ , TAuditReportBase<YandexQuery::ControlQueryRequest, YandexQuery::Query> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // ModifyQuery
+ using ModifyQueryAuditReport = TAuditReport<YandexQuery::ModifyQueryRequest, YandexQuery::Query>;
+
+ template<>
+ struct TAuditReport<YandexQuery::ModifyQueryRequest, YandexQuery::Query>
+ : NActors::TEventLocal<ModifyQueryAuditReport, EvModifyQueryReport>
+ , TAuditReportBase<YandexQuery::ModifyQueryRequest, YandexQuery::Query> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+ // DeleteQuery
+ using DeleteQueryAuditReport = TAuditReport<YandexQuery::DeleteQueryRequest, YandexQuery::Query>;
+
+ template<>
+ struct TAuditReport<YandexQuery::DeleteQueryRequest, YandexQuery::Query>
+ : NActors::TEventLocal<DeleteQueryAuditReport, EvDeleteQueryReport>
+ , TAuditReportBase<YandexQuery::DeleteQueryRequest, YandexQuery::Query> {
+ using TAuditReportBase::TAuditReportBase;
+ };
+
+
+ template <class TRequest, class TAuditDetailsObj>
+ static TAuditReport<TRequest, TAuditDetailsObj>* MakeAuditEvent(
+ TExtraInfo&& extraInfo,
+ const TRequest& request,
+ const NYql::TIssues& issues,
+ const TAuditDetails<TAuditDetailsObj>& details,
+ std::optional<TString> eventId = std::nullopt) {
+ return new TAuditReport<TRequest, TAuditDetailsObj>(std::move(extraInfo), request, issues, details, eventId);
+ }
+};
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/audit/events/ya.make b/ydb/core/yq/libs/audit/events/ya.make
index 44fb815a370..cb36076615b 100644
--- a/ydb/core/yq/libs/audit/events/ya.make
+++ b/ydb/core/yq/libs/audit/events/ya.make
@@ -1,18 +1,18 @@
OWNER(g:yq)
-
-LIBRARY()
-
-SRCS(
- events.cpp
-)
-
-PEERDIR(
- library/cpp/actors/core
- library/cpp/actors/interconnect
+
+LIBRARY()
+
+SRCS(
+ events.cpp
+)
+
+PEERDIR(
+ library/cpp/actors/core
+ library/cpp/actors/interconnect
ydb/core/yq/libs/checkpointing_common
ydb/core/yq/libs/control_plane_storage/events
ydb/public/api/protos
ydb/library/yql/public/issue
-)
-
-END()
+)
+
+END()
diff --git a/ydb/core/yq/libs/audit/mock/ya.make b/ydb/core/yq/libs/audit/mock/ya.make
index 3f7e5f017d7..2d8dc3ebe5d 100644
--- a/ydb/core/yq/libs/audit/mock/ya.make
+++ b/ydb/core/yq/libs/audit/mock/ya.make
@@ -1,14 +1,14 @@
OWNER(g:yq)
-
-LIBRARY()
-
-SRCS(
- yq_mock_audit_service.cpp
-)
-
-PEERDIR(
+
+LIBRARY()
+
+SRCS(
+ yq_mock_audit_service.cpp
+)
+
+PEERDIR(
ydb/core/yq/libs/audit/events
ydb/core/yq/libs/config/protos
-)
-
-END()
+)
+
+END()
diff --git a/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp b/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp
index cbe5a91b272..74f994e0df9 100644
--- a/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp
+++ b/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.cpp
@@ -1,75 +1,75 @@
-#include "yq_mock_audit_service.h"
-
+#include "yq_mock_audit_service.h"
+
#include <ydb/core/yq/libs/audit/events/events.h>
-
-#include <library/cpp/actors/core/hfunc.h>
-
-namespace NYq {
-
-class TYqMockAuditServiceActor : public NActors::TActor<TYqMockAuditServiceActor> {
-public:
- TYqMockAuditServiceActor() : TActor<TYqMockAuditServiceActor>(&TYqMockAuditServiceActor::StateFunc) {}
-
+
+#include <library/cpp/actors/core/hfunc.h>
+
+namespace NYq {
+
+class TYqMockAuditServiceActor : public NActors::TActor<TYqMockAuditServiceActor> {
+public:
+ TYqMockAuditServiceActor() : TActor<TYqMockAuditServiceActor>(&TYqMockAuditServiceActor::StateFunc) {}
+
static constexpr char ActorName[] = "YQ_MOCK_AUDIT_SERVICE";
-private:
- STRICT_STFUNC(StateFunc,
- hFunc(TEvAuditService::CreateBindingAuditReport, Handle);
- hFunc(TEvAuditService::ModifyBindingAuditReport, Handle);
- hFunc(TEvAuditService::DeleteBindingAuditReport, Handle);
- hFunc(TEvAuditService::CreateConnectionAuditReport, Handle);
- hFunc(TEvAuditService::ModifyConnectionAuditReport, Handle);
- hFunc(TEvAuditService::DeleteConnectionAuditReport, Handle);
- hFunc(TEvAuditService::CreateQueryAuditReport, Handle);
- hFunc(TEvAuditService::ControlQueryAuditReport, Handle);
- hFunc(TEvAuditService::ModifyQueryAuditReport, Handle);
- hFunc(TEvAuditService::DeleteQueryAuditReport, Handle);
- )
-
- void Handle(TEvAuditService::CreateBindingAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::ModifyBindingAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::DeleteBindingAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::CreateConnectionAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::ModifyConnectionAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::DeleteConnectionAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::CreateQueryAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::ControlQueryAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::ModifyQueryAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-
- void Handle(TEvAuditService::DeleteQueryAuditReport::TPtr& ev) {
- Y_UNUSED(ev);
- }
-};
-
+private:
+ STRICT_STFUNC(StateFunc,
+ hFunc(TEvAuditService::CreateBindingAuditReport, Handle);
+ hFunc(TEvAuditService::ModifyBindingAuditReport, Handle);
+ hFunc(TEvAuditService::DeleteBindingAuditReport, Handle);
+ hFunc(TEvAuditService::CreateConnectionAuditReport, Handle);
+ hFunc(TEvAuditService::ModifyConnectionAuditReport, Handle);
+ hFunc(TEvAuditService::DeleteConnectionAuditReport, Handle);
+ hFunc(TEvAuditService::CreateQueryAuditReport, Handle);
+ hFunc(TEvAuditService::ControlQueryAuditReport, Handle);
+ hFunc(TEvAuditService::ModifyQueryAuditReport, Handle);
+ hFunc(TEvAuditService::DeleteQueryAuditReport, Handle);
+ )
+
+ void Handle(TEvAuditService::CreateBindingAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::ModifyBindingAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::DeleteBindingAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::CreateConnectionAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::ModifyConnectionAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::DeleteConnectionAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::CreateQueryAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::ControlQueryAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::ModifyQueryAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+
+ void Handle(TEvAuditService::DeleteQueryAuditReport::TPtr& ev) {
+ Y_UNUSED(ev);
+ }
+};
+
NActors::IActor* CreateMockYqAuditServiceActor(const NConfig::TAuditConfig& config) {
- Y_UNUSED(config);
- return new TYqMockAuditServiceActor();
-}
-
-} // namespace NYq
+ Y_UNUSED(config);
+ return new TYqMockAuditServiceActor();
+}
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h b/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h
index 27c345420a5..f00bf089599 100644
--- a/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h
+++ b/ydb/core/yq/libs/audit/mock/yq_mock_audit_service.h
@@ -1,14 +1,14 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/yq/libs/config/protos/audit.pb.h>
-
-#include <library/cpp/actors/core/actor.h>
-
-#include <util/system/types.h>
-
-
-namespace NYq {
-
+
+#include <library/cpp/actors/core/actor.h>
+
+#include <util/system/types.h>
+
+
+namespace NYq {
+
NActors::IActor* CreateMockYqAuditServiceActor(const NConfig::TAuditConfig& config);
-
-} // namespace NYq
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/audit/ya.make b/ydb/core/yq/libs/audit/ya.make
index 947c8873bc1..a0671f84d47 100644
--- a/ydb/core/yq/libs/audit/ya.make
+++ b/ydb/core/yq/libs/audit/ya.make
@@ -1,14 +1,14 @@
OWNER(g:yq)
-
-LIBRARY()
-
-SRCS(
- yq_audit_service.cpp
-)
-
-END()
-
-RECURSE(
- events
- mock
-)
+
+LIBRARY()
+
+SRCS(
+ yq_audit_service.cpp
+)
+
+END()
+
+RECURSE(
+ events
+ mock
+)
diff --git a/ydb/core/yq/libs/audit/yq_audit_service.cpp b/ydb/core/yq/libs/audit/yq_audit_service.cpp
index 910422f276f..7e9be493360 100644
--- a/ydb/core/yq/libs/audit/yq_audit_service.cpp
+++ b/ydb/core/yq/libs/audit/yq_audit_service.cpp
@@ -1,13 +1,13 @@
-#include <library/cpp/actors/core/actor.h>
-
-#include <util/system/types.h>
-
-
-namespace NYq {
-
-NActors::TActorId YqAuditServiceActorId() {
- constexpr TStringBuf name = "YQAUDSVC";
- return NActors::TActorId(0, name);
-}
-
-} // namespace NYq
+#include <library/cpp/actors/core/actor.h>
+
+#include <util/system/types.h>
+
+
+namespace NYq {
+
+NActors::TActorId YqAuditServiceActorId() {
+ constexpr TStringBuf name = "YQAUDSVC";
+ return NActors::TActorId(0, name);
+}
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/audit/yq_audit_service.h b/ydb/core/yq/libs/audit/yq_audit_service.h
index 150938657a5..c750c4d09c0 100644
--- a/ydb/core/yq/libs/audit/yq_audit_service.h
+++ b/ydb/core/yq/libs/audit/yq_audit_service.h
@@ -1,12 +1,12 @@
-#pragma once
-
-#include <library/cpp/actors/core/actor.h>
-
-#include <util/system/types.h>
-
-
-namespace NYq {
-
-NActors::TActorId YqAuditServiceActorId();
-
-} // namespace NYq
+#pragma once
+
+#include <library/cpp/actors/core/actor.h>
+
+#include <util/system/types.h>
+
+
+namespace NYq {
+
+NActors::TActorId YqAuditServiceActorId();
+
+} // namespace NYq
diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp
index 220a37f233b..07c8d5101a0 100644
--- a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp
+++ b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.cpp
@@ -40,13 +40,13 @@ class TStorageProxy : public TActorBootstrapped<TStorageProxy> {
TCheckpointStoragePtr CheckpointStorage;
TStateStoragePtr StateStorage;
TActorId ActorGC;
- NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory;
+ NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory;
public:
- explicit TStorageProxy(
+ explicit TStorageProxy(
const NConfig::TCheckpointCoordinatorConfig& config,
const NConfig::TCommonConfig& commonConfig,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
void Bootstrap();
@@ -97,10 +97,10 @@ static void FillDefaultParameters(NConfig::TCheckpointCoordinatorConfig& checkpo
}
}
-TStorageProxy::TStorageProxy(
+TStorageProxy::TStorageProxy(
const NConfig::TCheckpointCoordinatorConfig& config,
const NConfig::TCommonConfig& commonConfig,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
: Config(config)
, CommonConfig(commonConfig)
, StorageConfig(Config.GetStorage())
@@ -390,11 +390,11 @@ void TStorageProxy::Handle(NYql::NDq::TEvDqCompute::TEvGetTaskState::TPtr& ev) {
////////////////////////////////////////////////////////////////////////////////
-std::unique_ptr<NActors::IActor> NewStorageProxy(
+std::unique_ptr<NActors::IActor> NewStorageProxy(
const NConfig::TCheckpointCoordinatorConfig& config,
const NConfig::TCommonConfig& commonConfig,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
-{
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
+{
return std::unique_ptr<NActors::IActor>(new TStorageProxy(config, commonConfig, credentialsProviderFactory));
}
diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.h b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.h
index f1b368f96f1..c67d928a151 100644
--- a/ydb/core/yq/libs/checkpoint_storage/storage_proxy.h
+++ b/ydb/core/yq/libs/checkpoint_storage/storage_proxy.h
@@ -4,16 +4,16 @@
#include <ydb/core/yq/libs/config/protos/common.pb.h>
#include <ydb/library/security/ydb_credentials_provider_factory.h>
-
+
#include <library/cpp/actors/core/actor.h>
#include <memory>
namespace NYq {
-std::unique_ptr<NActors::IActor> NewStorageProxy(
+std::unique_ptr<NActors::IActor> NewStorageProxy(
const NConfig::TCheckpointCoordinatorConfig& config,
const NConfig::TCommonConfig& commonConfig,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
} // namespace NYq
diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_service.cpp b/ydb/core/yq/libs/checkpoint_storage/storage_service.cpp
index b6825d8ef9f..421c19d7fdb 100644
--- a/ydb/core/yq/libs/checkpoint_storage/storage_service.cpp
+++ b/ydb/core/yq/libs/checkpoint_storage/storage_service.cpp
@@ -8,11 +8,11 @@ using namespace NActors;
////////////////////////////////////////////////////////////////////////////////
-std::unique_ptr<NActors::IActor> NewCheckpointStorageService(
+std::unique_ptr<NActors::IActor> NewCheckpointStorageService(
const NConfig::TCheckpointCoordinatorConfig& config,
const NConfig::TCommonConfig& commonConfig,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
-{
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
+{
return NewStorageProxy(config, commonConfig, credentialsProviderFactory);
}
diff --git a/ydb/core/yq/libs/checkpoint_storage/storage_service.h b/ydb/core/yq/libs/checkpoint_storage/storage_service.h
index 53f7cb8c430..5eab844545a 100644
--- a/ydb/core/yq/libs/checkpoint_storage/storage_service.h
+++ b/ydb/core/yq/libs/checkpoint_storage/storage_service.h
@@ -4,16 +4,16 @@
#include <ydb/core/yq/libs/config/protos/common.pb.h>
#include <ydb/library/security/ydb_credentials_provider_factory.h>
-
+
#include <library/cpp/actors/core/actor.h>
#include <memory>
namespace NYq {
-std::unique_ptr<NActors::IActor> NewCheckpointStorageService(
+std::unique_ptr<NActors::IActor> NewCheckpointStorageService(
const NConfig::TCheckpointCoordinatorConfig& config,
const NConfig::TCommonConfig& commonConfig,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
} // namespace NYq
diff --git a/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp b/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp
index 2e8acabc841..7799b52daaf 100644
--- a/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp
+++ b/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.cpp
@@ -552,8 +552,8 @@ class TCheckpointStorage : public ICheckpointStorage {
const NConfig::TYdbStorageConfig Config;
public:
- explicit TCheckpointStorage(
- const NConfig::TYdbStorageConfig& config,
+ explicit TCheckpointStorage(
+ const NConfig::TYdbStorageConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const IEntityIdGenerator::TPtr& entityIdGenerator);
@@ -621,11 +621,11 @@ private:
////////////////////////////////////////////////////////////////////////////////
-TCheckpointStorage::TCheckpointStorage(
- const NConfig::TYdbStorageConfig& config,
+TCheckpointStorage::TCheckpointStorage(
+ const NConfig::TYdbStorageConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const IEntityIdGenerator::TPtr& entityIdGenerator)
- : YdbConnection(NewYdbConnection(config, credentialsProviderFactory))
+ : YdbConnection(NewYdbConnection(config, credentialsProviderFactory))
, Config(config)
, EntityIdGenerator(entityIdGenerator)
{
@@ -1181,8 +1181,8 @@ TExecDataQuerySettings NYq::TCheckpointStorage::DefaultExecDataQuerySettings() {
////////////////////////////////////////////////////////////////////////////////
-TCheckpointStoragePtr NewYdbCheckpointStorage(
- const NConfig::TYdbStorageConfig& config,
+TCheckpointStoragePtr NewYdbCheckpointStorage(
+ const NConfig::TYdbStorageConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const IEntityIdGenerator::TPtr& entityIdGenerator)
{
diff --git a/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h b/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h
index 93d95f9be5f..c52b006f916 100644
--- a/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h
+++ b/ydb/core/yq/libs/checkpoint_storage/ydb_checkpoint_storage.h
@@ -10,8 +10,8 @@ namespace NYq {
////////////////////////////////////////////////////////////////////////////////
-TCheckpointStoragePtr NewYdbCheckpointStorage(
- const NConfig::TYdbStorageConfig& config,
+TCheckpointStoragePtr NewYdbCheckpointStorage(
+ const NConfig::TYdbStorageConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const IEntityIdGenerator::TPtr& entityIdGenerator);
diff --git a/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp b/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp
index d7050a003be..ab209ec1e89 100644
--- a/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp
+++ b/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.cpp
@@ -150,9 +150,9 @@ class TStateStorage : public IStateStorage {
const NConfig::TYdbStorageConfig Config;
public:
- explicit TStateStorage(
- const NConfig::TYdbStorageConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
+ explicit TStateStorage(
+ const NConfig::TYdbStorageConfig& config,
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
~TStateStorage() = default;
TFuture<TIssues> Init() override;
@@ -186,10 +186,10 @@ public:
////////////////////////////////////////////////////////////////////////////////
-TStateStorage::TStateStorage(
- const NConfig::TYdbStorageConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
- : YdbConnection(NewYdbConnection(config, credentialsProviderFactory))
+TStateStorage::TStateStorage(
+ const NConfig::TYdbStorageConfig& config,
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
+ : YdbConnection(NewYdbConnection(config, credentialsProviderFactory))
, Config(config)
{
}
@@ -554,11 +554,11 @@ TFuture<TStatus> TStateStorage::UpsertState(const TContextPtr& context) {
////////////////////////////////////////////////////////////////////////////////
-TStateStoragePtr NewYdbStateStorage(
- const NConfig::TYdbStorageConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
+TStateStoragePtr NewYdbStateStorage(
+ const NConfig::TYdbStorageConfig& config,
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory)
{
- return new TStateStorage(config, credentialsProviderFactory);
+ return new TStateStorage(config, credentialsProviderFactory);
}
} // namespace NYq
diff --git a/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h b/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h
index 4faf4f17715..61dba973de7 100644
--- a/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h
+++ b/ydb/core/yq/libs/checkpoint_storage/ydb_state_storage.h
@@ -9,8 +9,8 @@ namespace NYq {
////////////////////////////////////////////////////////////////////////////////
-TStateStoragePtr NewYdbStateStorage(
- const NConfig::TYdbStorageConfig& config,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
+TStateStoragePtr NewYdbStateStorage(
+ const NConfig::TYdbStorageConfig& config,
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
} // namespace NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/control_plane_storage.h b/ydb/core/yq/libs/control_plane_storage/control_plane_storage.h
index ba38450e984..5180aedeea7 100644
--- a/ydb/core/yq/libs/control_plane_storage/control_plane_storage.h
+++ b/ydb/core/yq/libs/control_plane_storage/control_plane_storage.h
@@ -38,11 +38,11 @@ NActors::TActorId ControlPlaneStorageServiceActorId(ui32 nodeId = 0);
NActors::IActor* CreateInMemoryControlPlaneStorageServiceActor(const NConfig::TControlPlaneStorageConfig& config);
-NActors::IActor* CreateYdbControlPlaneStorageServiceActor(
+NActors::IActor* CreateYdbControlPlaneStorageServiceActor(
const NConfig::TControlPlaneStorageConfig& config,
const NConfig::TCommonConfig& common,
- const NMonitoring::TDynamicCounterPtr& counters,
- const NYq::TYqSharedResources::TPtr& yqSharedResources,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
+ const NMonitoring::TDynamicCounterPtr& counters,
+ const NYq::TYqSharedResources::TPtr& yqSharedResources,
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory);
} // namespace NYq
diff --git a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp
index a91809bbe91..783b9242385 100644
--- a/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/in_memory_control_plane_storage.cpp
@@ -79,18 +79,18 @@ private:
hFunc(TEvControlPlaneStorage::TEvDescribeBindingRequest, Handle);
hFunc(TEvControlPlaneStorage::TEvModifyBindingRequest, Handle);
hFunc(TEvControlPlaneStorage::TEvDeleteBindingRequest, Handle);
- hFunc(TEvControlPlaneStorage::TEvDescribeJobRequest, Handle);
- hFunc(TEvControlPlaneStorage::TEvWriteResultDataRequest, Handle);
- hFunc(TEvControlPlaneStorage::TEvGetTaskRequest, Handle);
- hFunc(TEvControlPlaneStorage::TEvPingTaskRequest, Handle);
- hFunc(TEvControlPlaneStorage::TEvNodesHealthCheckRequest, Handle);
- hFunc(NActors::NMon::TEvHttpInfo, Handle);
+ hFunc(TEvControlPlaneStorage::TEvDescribeJobRequest, Handle);
+ hFunc(TEvControlPlaneStorage::TEvWriteResultDataRequest, Handle);
+ hFunc(TEvControlPlaneStorage::TEvGetTaskRequest, Handle);
+ hFunc(TEvControlPlaneStorage::TEvPingTaskRequest, Handle);
+ hFunc(TEvControlPlaneStorage::TEvNodesHealthCheckRequest, Handle);
+ hFunc(NActors::NMon::TEvHttpInfo, Handle);
)
void Handle(TEvControlPlaneStorage::TEvCreateQueryRequest::TPtr& ev)
{
- CPS_LOG_I("CreateQueryRequest");
-
+ CPS_LOG_I("CreateQueryRequest");
+
const YandexQuery::CreateQueryRequest& request = ev->Get()->Request;
CPS_LOG_D("CreateQueryRequest: " << request.DebugString());
CleanupIndempotencyKeys();
@@ -138,25 +138,25 @@ private:
void Handle(TEvControlPlaneStorage::TEvListQueriesRequest::TPtr& ev)
{
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvListQueriesRequest::TPtr,
- YandexQuery::ListQueriesResult,
- TEvControlPlaneStorage::TEvListQueriesResponse>(ev, "ListQueriesRequest");
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvListQueriesRequest::TPtr,
+ YandexQuery::ListQueriesResult,
+ TEvControlPlaneStorage::TEvListQueriesResponse>(ev, "ListQueriesRequest");
}
void Handle(TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr& ev)
{
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr,
- YandexQuery::DescribeQueryResult,
- TEvControlPlaneStorage::TEvDescribeQueryResponse>(ev, "DescribeQueryRequest");
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvDescribeQueryRequest::TPtr,
+ YandexQuery::DescribeQueryResult,
+ TEvControlPlaneStorage::TEvDescribeQueryResponse>(ev, "DescribeQueryRequest");
}
void Handle(TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr,
- YandexQuery::ModifyQueryResult,
+ TEvControlPlaneStorage::TEvModifyQueryRequest::TPtr,
+ YandexQuery::ModifyQueryResult,
TEvControlPlaneStorage::TEvModifyQueryResponse,
TAuditDetails<YandexQuery::Query>>(ev, "ModifyQueryRequest");
}
@@ -164,8 +164,8 @@ private:
void Handle(TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr,
- YandexQuery::DeleteQueryResult,
+ TEvControlPlaneStorage::TEvDeleteQueryRequest::TPtr,
+ YandexQuery::DeleteQueryResult,
TEvControlPlaneStorage::TEvDeleteQueryResponse,
TAuditDetails<YandexQuery::Query>>(ev, "DeleteQueryRequest");
}
@@ -173,58 +173,58 @@ private:
void Handle(TEvControlPlaneStorage::TEvControlQueryRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvControlQueryRequest::TPtr,
- YandexQuery::ControlQueryResult,
+ TEvControlPlaneStorage::TEvControlQueryRequest::TPtr,
+ YandexQuery::ControlQueryResult,
TEvControlPlaneStorage::TEvControlQueryResponse,
TAuditDetails<YandexQuery::Query>>(ev, "ControlQueryRequest");
}
void Handle(TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr& ev)
{
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr,
- YandexQuery::GetResultDataResult,
- TEvControlPlaneStorage::TEvGetResultDataResponse>(ev, "GetResultDataRequest");
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvGetResultDataRequest::TPtr,
+ YandexQuery::GetResultDataResult,
+ TEvControlPlaneStorage::TEvGetResultDataResponse>(ev, "GetResultDataRequest");
}
void Handle(TEvControlPlaneStorage::TEvListJobsRequest::TPtr& ev)
{
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvListJobsRequest::TPtr,
- YandexQuery::ListJobsResult,
- TEvControlPlaneStorage::TEvListJobsResponse>(ev, "ListJobsRequest");
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvListJobsRequest::TPtr,
+ YandexQuery::ListJobsResult,
+ TEvControlPlaneStorage::TEvListJobsResponse>(ev, "ListJobsRequest");
}
void Handle(TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr,
- YandexQuery::CreateConnectionResult,
+ TEvControlPlaneStorage::TEvCreateConnectionRequest::TPtr,
+ YandexQuery::CreateConnectionResult,
TEvControlPlaneStorage::TEvCreateConnectionResponse,
TAuditDetails<YandexQuery::Connection>>(ev, "CreateConnectionRequest");
}
void Handle(TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr& ev)
{
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr,
- YandexQuery::ListConnectionsResult,
- TEvControlPlaneStorage::TEvListConnectionsResponse>(ev, "ListConnectionsRequest");
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvListConnectionsRequest::TPtr,
+ YandexQuery::ListConnectionsResult,
+ TEvControlPlaneStorage::TEvListConnectionsResponse>(ev, "ListConnectionsRequest");
}
void Handle(TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr& ev)
{
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr,
- YandexQuery::DescribeConnectionResult,
- TEvControlPlaneStorage::TEvDescribeConnectionResponse>(ev, "DescribeConnectionRequest");
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvDescribeConnectionRequest::TPtr,
+ YandexQuery::DescribeConnectionResult,
+ TEvControlPlaneStorage::TEvDescribeConnectionResponse>(ev, "DescribeConnectionRequest");
}
void Handle(TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr,
- YandexQuery::ModifyConnectionResult,
+ TEvControlPlaneStorage::TEvModifyConnectionRequest::TPtr,
+ YandexQuery::ModifyConnectionResult,
TEvControlPlaneStorage::TEvModifyConnectionResponse,
TAuditDetails<YandexQuery::Connection>>(ev, "ModifyConnectionRequest");
}
@@ -232,8 +232,8 @@ private:
void Handle(TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr,
- YandexQuery::DeleteConnectionResult,
+ TEvControlPlaneStorage::TEvDeleteConnectionRequest::TPtr,
+ YandexQuery::DeleteConnectionResult,
TEvControlPlaneStorage::TEvDeleteConnectionResponse,
TAuditDetails<YandexQuery::Connection>>(ev, "DeleteConnectionRequest");
}
@@ -241,33 +241,33 @@ private:
void Handle(TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr,
- YandexQuery::CreateBindingResult,
+ TEvControlPlaneStorage::TEvCreateBindingRequest::TPtr,
+ YandexQuery::CreateBindingResult,
TEvControlPlaneStorage::TEvCreateBindingResponse,
TAuditDetails<YandexQuery::Binding>>(ev, "CreateBindingRequest");
}
void Handle(TEvControlPlaneStorage::TEvListBindingsRequest::TPtr& ev)
{
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvListBindingsRequest::TPtr,
- YandexQuery::ListBindingsResult,
- TEvControlPlaneStorage::TEvListBindingsResponse>(ev, "ListBindingsRequest");
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvListBindingsRequest::TPtr,
+ YandexQuery::ListBindingsResult,
+ TEvControlPlaneStorage::TEvListBindingsResponse>(ev, "ListBindingsRequest");
}
void Handle(TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr& ev)
{
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr,
- YandexQuery::DescribeBindingResult,
- TEvControlPlaneStorage::TEvDescribeBindingResponse>(ev, "DescribeBindingRequest");
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvDescribeBindingRequest::TPtr,
+ YandexQuery::DescribeBindingResult,
+ TEvControlPlaneStorage::TEvDescribeBindingResponse>(ev, "DescribeBindingRequest");
}
void Handle(TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr,
- YandexQuery::ModifyBindingResult,
+ TEvControlPlaneStorage::TEvModifyBindingRequest::TPtr,
+ YandexQuery::ModifyBindingResult,
TEvControlPlaneStorage::TEvModifyBindingResponse,
TAuditDetails<YandexQuery::Binding>>(ev, "ModifyBindingRequest");
}
@@ -275,67 +275,67 @@ private:
void Handle(TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr& ev)
{
SendEmptyAuditResponse<
- TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr,
- YandexQuery::DeleteBindingResult,
+ TEvControlPlaneStorage::TEvDeleteBindingRequest::TPtr,
+ YandexQuery::DeleteBindingResult,
TEvControlPlaneStorage::TEvDeleteBindingResponse,
TAuditDetails<YandexQuery::Binding>>(ev, "DeleteBindingRequest");
}
- void Handle(TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr& ev)
- {
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr,
- YandexQuery::DescribeJobResult,
- TEvControlPlaneStorage::TEvDescribeJobResponse>(ev, "DescribeJobRequest");
- }
-
- void Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev)
- {
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr,
- NYql::TIssues,
- TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev, "WriteResultDataRequest");
- }
-
- void Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev)
- {
- CPS_LOG_I("GetTaskRequest");
- TVector<TEvControlPlaneStorage::TTask> tasks;
- TString owner;
- auto event = std::make_unique<TEvControlPlaneStorage::TEvGetTaskResponse>(tasks, owner);
- NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie));
- }
-
- void Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev)
- {
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvPingTaskRequest::TPtr,
- YandexQuery::QueryAction,
- TEvControlPlaneStorage::TEvPingTaskResponse>(ev, "PingTaskRequest");
- }
-
- void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev)
- {
- SendEmptyResponse<
- TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr,
+ void Handle(TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr& ev)
+ {
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvDescribeJobRequest::TPtr,
+ YandexQuery::DescribeJobResult,
+ TEvControlPlaneStorage::TEvDescribeJobResponse>(ev, "DescribeJobRequest");
+ }
+
+ void Handle(TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr& ev)
+ {
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvWriteResultDataRequest::TPtr,
+ NYql::TIssues,
+ TEvControlPlaneStorage::TEvWriteResultDataResponse>(ev, "WriteResultDataRequest");
+ }
+
+ void Handle(TEvControlPlaneStorage::TEvGetTaskRequest::TPtr& ev)
+ {
+ CPS_LOG_I("GetTaskRequest");
+ TVector<TEvControlPlaneStorage::TTask> tasks;
+ TString owner;
+ auto event = std::make_unique<TEvControlPlaneStorage::TEvGetTaskResponse>(tasks, owner);
+ NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie));
+ }
+
+ void Handle(TEvControlPlaneStorage::TEvPingTaskRequest::TPtr& ev)
+ {
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvPingTaskRequest::TPtr,
+ YandexQuery::QueryAction,
+ TEvControlPlaneStorage::TEvPingTaskResponse>(ev, "PingTaskRequest");
+ }
+
+ void Handle(TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr& ev)
+ {
+ SendEmptyResponse<
+ TEvControlPlaneStorage::TEvNodesHealthCheckRequest::TPtr,
Yq::Private::NodesHealthCheckResult,
- TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev, "NodesHealthCheckRequest");
- }
-
- void Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) {
- TStringStream str;
- Send(ev->Sender, new NActors::NMon::TEvHttpInfoRes(str.Str()));
- }
-
- template<typename TRequest, typename TResult, typename TEvResult>
- void SendEmptyResponse(TRequest& ev, std::string logText) {
- CPS_LOG_I(logText);
-
+ TEvControlPlaneStorage::TEvNodesHealthCheckResponse>(ev, "NodesHealthCheckRequest");
+ }
+
+ void Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) {
+ TStringStream str;
+ Send(ev->Sender, new NActors::NMon::TEvHttpInfoRes(str.Str()));
+ }
+
+ template<typename TRequest, typename TResult, typename TEvResult>
+ void SendEmptyResponse(TRequest& ev, std::string logText) {
+ CPS_LOG_I(logText);
+
TResult result = {};
- auto event = std::make_unique<TEvResult>(result);
- NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie));
- }
-
+ auto event = std::make_unique<TEvResult>(result);
+ NActors::TActivationContext::ActorSystem()->Send(new IEventHandle(ev->Sender, SelfId(), event.release(), 0, ev->Cookie));
+ }
+
template<typename TRequest, typename TResult, typename TEvResult, typename TAuditDetails>
void SendEmptyAuditResponse(TRequest& ev, std::string logText) {
CPS_LOG_I(logText);
diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp
index b41fa5f37b9..222ebb6e8e3 100644
--- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp
+++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage.cpp
@@ -3,7 +3,7 @@
#include "ydb_control_plane_storage_impl.h"
#include <ydb/library/security/ydb_credentials_provider_factory.h>
-
+
namespace NYq {
namespace {
@@ -670,12 +670,12 @@ TAsyncStatus TYdbControlPlaneStorageActor::ReadModifyWrite(
return promise.GetFuture();
}
-NActors::IActor* CreateYdbControlPlaneStorageServiceActor(
+NActors::IActor* CreateYdbControlPlaneStorageServiceActor(
const NConfig::TControlPlaneStorageConfig& config,
const NConfig::TCommonConfig& common,
- const NMonitoring::TDynamicCounterPtr& counters,
- const ::NYq::TYqSharedResources::TPtr& yqSharedResources,
- const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) {
+ const NMonitoring::TDynamicCounterPtr& counters,
+ const ::NYq::TYqSharedResources::TPtr& yqSharedResources,
+ const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory) {
return new TYdbControlPlaneStorageActor(config, common, counters, yqSharedResources, credentialsProviderFactory);
}
diff --git a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h
index 34f9fc5c710..1c8cda7ee72 100644
--- a/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h
+++ b/ydb/core/yq/libs/control_plane_storage/ydb_control_plane_storage_impl.h
@@ -183,20 +183,20 @@ class TYdbControlPlaneStorageActor : public NActors::TActorBootstrapped<TYdbCont
static constexpr int64_t InitialRevision = 1;
- NKikimr::TYdbCredentialsProviderFactory CredProviderFactory;
-
+ NKikimr::TYdbCredentialsProviderFactory CredProviderFactory;
+
public:
- TYdbControlPlaneStorageActor(
+ TYdbControlPlaneStorageActor(
const NConfig::TControlPlaneStorageConfig& config,
const NConfig::TCommonConfig& common,
- const NMonitoring::TDynamicCounterPtr& counters,
- const ::NYq::TYqSharedResources::TPtr& yqSharedResources,
- const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory)
+ const NMonitoring::TDynamicCounterPtr& counters,
+ const ::NYq::TYqSharedResources::TPtr& yqSharedResources,
+ const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory)
: Counters(counters)
, FinalStatusCounters(counters)
, Config(config, common)
, YqSharedResources(yqSharedResources)
- , CredProviderFactory(credProviderFactory)
+ , CredProviderFactory(credProviderFactory)
{
}
diff --git a/ydb/core/yq/libs/events/event_subspace.h b/ydb/core/yq/libs/events/event_subspace.h
index 9eb8b2f3a66..99814caaae0 100644
--- a/ydb/core/yq/libs/events/event_subspace.h
+++ b/ydb/core/yq/libs/events/event_subspace.h
@@ -24,7 +24,7 @@ struct TYqEventSubspace {
ConfigUpdater,
ControlPlaneStorage,
ControlPlaneProxy,
- AuditService,
+ AuditService,
TestConnection,
SubspacesEnd,
diff --git a/ydb/core/yq/libs/init/init.cpp b/ydb/core/yq/libs/init/init.cpp
index fc4ad80f00d..d66ac202bfa 100644
--- a/ydb/core/yq/libs/init/init.cpp
+++ b/ydb/core/yq/libs/init/init.cpp
@@ -103,7 +103,7 @@ void Init(
const TString& tenant,
::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
const IYqSharedResources::TPtr& iyqSharedResources,
- const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory,
+ const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory,
const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const ui32& icPort
@@ -124,7 +124,7 @@ void Init(
credentialsProviderFactory);
actorRegistrator(NYq::ControlPlaneStorageServiceActorId(), controlPlaneStorage);
}
-
+
if (protoConfig.GetTestConnection().GetEnabled()) {
auto testConnection = NYq::CreateTestConnectionActor(
protoConfig.GetTestConnection(),
@@ -270,11 +270,11 @@ void Init(
}
}
-IYqSharedResources::TPtr CreateYqSharedResources(
+IYqSharedResources::TPtr CreateYqSharedResources(
const NYq::NConfig::TConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters)
-{
+{
return CreateYqSharedResourcesImpl(config, credentialsProviderFactory, counters);
}
diff --git a/ydb/core/yq/libs/init/init.h b/ydb/core/yq/libs/init/init.h
index 1487fc950bd..fa26651f185 100644
--- a/ydb/core/yq/libs/init/init.h
+++ b/ydb/core/yq/libs/init/init.h
@@ -21,7 +21,7 @@ namespace NYq {
using TActorRegistrator = std::function<void(NActors::TActorId, NActors::IActor*)>;
-IYqSharedResources::TPtr CreateYqSharedResources(
+IYqSharedResources::TPtr CreateYqSharedResources(
const NYq::NConfig::TConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters);
@@ -34,7 +34,7 @@ void Init(
const TString& tenant,
::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections,
const IYqSharedResources::TPtr& yqSharedResources,
- const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory,
+ const std::function<IActor*(const NKikimrProto::NFolderService::TFolderServiceConfig& authConfig)>& folderServiceFactory,
const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const ui32& icPort
diff --git a/ydb/core/yq/libs/shared_resources/db_pool.cpp b/ydb/core/yq/libs/shared_resources/db_pool.cpp
index 125802a0d01..372dfd72cdf 100644
--- a/ydb/core/yq/libs/shared_resources/db_pool.cpp
+++ b/ydb/core/yq/libs/shared_resources/db_pool.cpp
@@ -221,22 +221,22 @@ static void PrepareConfig(NYq::NConfig::TDbPoolConfig& config) {
}
}
-TDbPoolMap::TDbPoolMap(
+TDbPoolMap::TDbPoolMap(
const NYq::NConfig::TDbPoolConfig& config,
- NYdb::TDriver driver,
+ NYdb::TDriver driver,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters)
: Config(config)
, Driver(driver)
- , CredentialsProviderFactory(credentialsProviderFactory)
+ , CredentialsProviderFactory(credentialsProviderFactory)
, Counters(counters)
{
PrepareConfig(Config);
}
-TDbPoolHolder::TDbPoolHolder(
+TDbPoolHolder::TDbPoolHolder(
const NYq::NConfig::TDbPoolConfig& config,
- const NYdb::TDriver& driver,
+ const NYdb::TDriver& driver,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters)
: Driver(driver)
@@ -286,12 +286,12 @@ TDbPool::TPtr TDbPoolMap::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) {
.Database(Config.GetStorage().GetDatabase())
.DiscoveryEndpoint(Config.GetStorage().GetEndpoint())
.DiscoveryMode(NYdb::EDiscoveryMode::Async);
- NKikimr::TYdbCredentialsSettings credSettings;
+ NKikimr::TYdbCredentialsSettings credSettings;
credSettings.UseLocalMetadata = Config.GetStorage().GetUseLocalMetadataService();
credSettings.OAuthToken = Config.GetStorage().GetToken();
-
- clientSettings.CredentialsProviderFactory(CredentialsProviderFactory(credSettings));
-
+
+ clientSettings.CredentialsProviderFactory(CredentialsProviderFactory(credSettings));
+
clientSettings.EnableSsl(Config.GetStorage().GetUseSsl());
TableClient = MakeHolder<NYdb::NTable::TTableClient>(Driver, clientSettings);
diff --git a/ydb/core/yq/libs/shared_resources/db_pool.h b/ydb/core/yq/libs/shared_resources/db_pool.h
index 6e6dec32e64..68d9adc2fca 100644
--- a/ydb/core/yq/libs/shared_resources/db_pool.h
+++ b/ydb/core/yq/libs/shared_resources/db_pool.h
@@ -5,7 +5,7 @@
#include <ydb/core/yq/libs/events/events.h>
#include <ydb/library/security/ydb_credentials_provider_factory.h>
-
+
#include <library/cpp/actors/core/actor.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
@@ -57,16 +57,16 @@ private:
NYdb::TDriver Driver;
THashMap<EDbPoolId, TDbPool::TPtr> Pools;
THolder<NYdb::NTable::TTableClient> TableClient;
- NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory;
+ NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory;
const NMonitoring::TDynamicCounterPtr Counters;
};
class TDbPoolHolder: public TThrRefBase {
public:
using TPtr = TIntrusivePtr<TDbPoolHolder>;
- TDbPoolHolder(
+ TDbPoolHolder(
const NYq::NConfig::TDbPoolConfig& config,
- const NYdb::TDriver& driver,
+ const NYdb::TDriver& driver,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters);
diff --git a/ydb/core/yq/libs/shared_resources/shared_resources.cpp b/ydb/core/yq/libs/shared_resources/shared_resources.cpp
index f84c3dfc4cd..3dfd566232f 100644
--- a/ydb/core/yq/libs/shared_resources/shared_resources.cpp
+++ b/ydb/core/yq/libs/shared_resources/shared_resources.cpp
@@ -84,7 +84,7 @@ struct TActorSystemPtrMixin {
};
struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedResources {
- explicit TYqSharedResourcesImpl(
+ explicit TYqSharedResourcesImpl(
const NYq::NConfig::TConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters)
@@ -118,7 +118,7 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes
return cfg;
}
- void CreateDbPoolHolder(
+ void CreateDbPoolHolder(
const NYq::NConfig::TDbPoolConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters) {
@@ -128,7 +128,7 @@ struct TYqSharedResourcesImpl : public TActorSystemPtrMixin, public TYqSharedRes
} // namespace
-TYqSharedResources::TPtr CreateYqSharedResourcesImpl(
+TYqSharedResources::TPtr CreateYqSharedResourcesImpl(
const NYq::NConfig::TConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters) {
diff --git a/ydb/core/yq/libs/shared_resources/shared_resources.h b/ydb/core/yq/libs/shared_resources/shared_resources.h
index 3f047e653af..bfd1ec66953 100644
--- a/ydb/core/yq/libs/shared_resources/shared_resources.h
+++ b/ydb/core/yq/libs/shared_resources/shared_resources.h
@@ -6,7 +6,7 @@
#include <ydb/core/yq/libs/shared_resources/interface/shared_resources.h>
#include <ydb/library/security/ydb_credentials_provider_factory.h>
-
+
#include <library/cpp/actors/core/actorsystem.h>
namespace NYq {
@@ -23,7 +23,7 @@ protected:
explicit TYqSharedResources(NYdb::TDriver driver);
};
-TYqSharedResources::TPtr CreateYqSharedResourcesImpl(
+TYqSharedResources::TPtr CreateYqSharedResourcesImpl(
const NYq::NConfig::TConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory,
const NMonitoring::TDynamicCounterPtr& counters);
diff --git a/ydb/core/yq/libs/ya.make b/ydb/core/yq/libs/ya.make
index 914fee971d1..ccc88a55b62 100644
--- a/ydb/core/yq/libs/ya.make
+++ b/ydb/core/yq/libs/ya.make
@@ -2,7 +2,7 @@ OWNER(g:yq)
RECURSE(
actors
- audit
+ audit
checkpoint_storage
checkpointing
checkpointing_common
diff --git a/ydb/core/yq/libs/ydb/ydb.cpp b/ydb/core/yq/libs/ydb/ydb.cpp
index 4230c93d00c..d6096abce32 100644
--- a/ydb/core/yq/libs/ydb/ydb.cpp
+++ b/ydb/core/yq/libs/ydb/ydb.cpp
@@ -205,14 +205,14 @@ NYdb::TDriverConfig GetDriverConfig(const NConfig::TYdbStorageConfig& config,
.SetEndpoint(config.GetEndpoint())
.SetDatabase(config.GetDatabase());
- NKikimr::TYdbCredentialsSettings credSettings;
- credSettings.UseLocalMetadata = config.GetUseLocalMetadataService();
- credSettings.OAuthToken = oauth;
- credSettings.SaKeyFile = config.GetSaKeyFile();
- credSettings.IamEndpoint = config.GetIamEndpoint();
-
- driverConfig.SetCredentialsProviderFactory(credProviderFactory(credSettings));
-
+ NKikimr::TYdbCredentialsSettings credSettings;
+ credSettings.UseLocalMetadata = config.GetUseLocalMetadataService();
+ credSettings.OAuthToken = oauth;
+ credSettings.SaKeyFile = config.GetSaKeyFile();
+ credSettings.IamEndpoint = config.GetIamEndpoint();
+
+ driverConfig.SetCredentialsProviderFactory(credProviderFactory(credSettings));
+
if (config.GetUseLocalMetadataService()) {
driverConfig.UseSecureConnection();
}
@@ -227,7 +227,7 @@ NYdb::TDriverConfig GetDriverConfig(const NConfig::TYdbStorageConfig& config,
TYdbConnectionPtr NewYdbConnection(const NConfig::TYdbStorageConfig& config,
const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory) {
- auto driverConfig = GetDriverConfig(config, credProviderFactory);
+ auto driverConfig = GetDriverConfig(config, credProviderFactory);
return MakeIntrusive<TYdbConnection>(driverConfig, config);
}
diff --git a/ydb/core/yq/libs/ydb/ydb.h b/ydb/core/yq/libs/ydb/ydb.h
index abe8d179090..14c07422883 100644
--- a/ydb/core/yq/libs/ydb/ydb.h
+++ b/ydb/core/yq/libs/ydb/ydb.h
@@ -94,9 +94,9 @@ using TGenerationContextPtr = TIntrusivePtr<TGenerationContext>;
////////////////////////////////////////////////////////////////////////////////
-NYdb::TDriverConfig GetDriverConfig(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory);
+NYdb::TDriverConfig GetDriverConfig(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory);
-TYdbConnectionPtr NewYdbConnection(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory);
+TYdbConnectionPtr NewYdbConnection(const NConfig::TYdbStorageConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credProviderFactory);
NYdb::TStatus MakeErrorStatus(
NYdb::EStatus code,
diff --git a/ydb/library/security/ya.make b/ydb/library/security/ya.make
index 3fd5c726d0e..ce048987e4a 100644
--- a/ydb/library/security/ya.make
+++ b/ydb/library/security/ya.make
@@ -12,7 +12,7 @@ PEERDIR(
SRCS(
util.cpp
- ydb_credentials_provider_factory.cpp
+ ydb_credentials_provider_factory.cpp
)
END()
diff --git a/ydb/library/security/ydb_credentials_provider_factory.cpp b/ydb/library/security/ydb_credentials_provider_factory.cpp
index 010f16d3f27..25174da6f05 100644
--- a/ydb/library/security/ydb_credentials_provider_factory.cpp
+++ b/ydb/library/security/ydb_credentials_provider_factory.cpp
@@ -1,13 +1,13 @@
-#include "ydb_credentials_provider_factory.h"
-
-
-namespace NKikimr {
-
-std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateYdbCredentialsProviderFactory(const TYdbCredentialsSettings& settings)
-{
- return settings.OAuthToken
- ? NYdb::CreateOAuthCredentialsProviderFactory(settings.OAuthToken)
- : NYdb::CreateInsecureCredentialsProviderFactory();
-}
-
-}
+#include "ydb_credentials_provider_factory.h"
+
+
+namespace NKikimr {
+
+std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateYdbCredentialsProviderFactory(const TYdbCredentialsSettings& settings)
+{
+ return settings.OAuthToken
+ ? NYdb::CreateOAuthCredentialsProviderFactory(settings.OAuthToken)
+ : NYdb::CreateInsecureCredentialsProviderFactory();
+}
+
+}
diff --git a/ydb/library/security/ydb_credentials_provider_factory.h b/ydb/library/security/ydb_credentials_provider_factory.h
index 3c0779de87f..92c986e2490 100644
--- a/ydb/library/security/ydb_credentials_provider_factory.h
+++ b/ydb/library/security/ydb_credentials_provider_factory.h
@@ -1,21 +1,21 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h>
-
-#include <memory>
-
-namespace NKikimr {
-
-struct TYdbCredentialsSettings {
- bool UseLocalMetadata = false;
- TString OAuthToken;
-
- TString SaKeyFile;
- TString IamEndpoint;
-};
-
-using TYdbCredentialsProviderFactory = std::function<std::shared_ptr<NYdb::ICredentialsProviderFactory>(const TYdbCredentialsSettings& settings)>;
-
-std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateYdbCredentialsProviderFactory(const TYdbCredentialsSettings& settings);
-
-}
+
+#include <memory>
+
+namespace NKikimr {
+
+struct TYdbCredentialsSettings {
+ bool UseLocalMetadata = false;
+ TString OAuthToken;
+
+ TString SaKeyFile;
+ TString IamEndpoint;
+};
+
+using TYdbCredentialsProviderFactory = std::function<std::shared_ptr<NYdb::ICredentialsProviderFactory>(const TYdbCredentialsSettings& settings)>;
+
+std::shared_ptr<NYdb::ICredentialsProviderFactory> CreateYdbCredentialsProviderFactory(const TYdbCredentialsSettings& settings);
+
+}
diff --git a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
index 143ecaab35a..215a1e9f692 100644
--- a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
@@ -731,11 +731,11 @@ TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, cons
auto traits = TCoHoppingTraits(hoppingSetting->Child(1));
auto timeExtractor = traits.TimeExtractor();
- auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
- for (const auto& usedField : usedType->GetItems()) {
- usedFields.insert(usedField->GetName());
- }
-
+ auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
+ for (const auto& usedField : usedType->GetItems()) {
+ usedFields.insert(usedField->GetName());
+ }
+
TSet<TStringBuf> lambdaSubset;
if (HaveFieldsSubset(timeExtractor.Body().Ptr(), *timeExtractor.Args().Arg(0).Raw(), lambdaSubset, parentsMap)) {
usedFields.insert(lambdaSubset.cbegin(), lambdaSubset.cend());
diff --git a/ydb/library/yql/core/common_opt/yql_co_flow2.cpp b/ydb/library/yql/core/common_opt/yql_co_flow2.cpp
index 754f4990a6c..88f91400319 100644
--- a/ydb/library/yql/core/common_opt/yql_co_flow2.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_flow2.cpp
@@ -67,11 +67,11 @@ TExprNode::TPtr AggregateSubsetFieldsAnalyzer(const TCoAggregate& node, TExprCon
auto traits = TCoHoppingTraits(hoppingSetting->Child(1));
auto timeExtractor = traits.TimeExtractor();
- auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
- for (const auto& usedField : usedType->GetItems()) {
- usedFields.insert(usedField->GetName());
- }
-
+ auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
+ for (const auto& usedField : usedType->GetItems()) {
+ usedFields.insert(usedField->GetName());
+ }
+
TSet<TStringBuf> lambdaSubset;
if (!HaveFieldsSubset(timeExtractor.Body().Ptr(), *timeExtractor.Args().Arg(0).Raw(), lambdaSubset, parentsMap)) {
return node.Ptr();
@@ -2019,42 +2019,42 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
.Build();
};
- map[TCoHoppingTraits::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- TCoHoppingTraits self(node);
-
- auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
-
- const auto lambdaBody = self.TimeExtractor().Body().Ptr();
- const auto& arg = self.TimeExtractor().Args().Arg(0).Ref();
-
- TSet<TStringBuf> usedFields;
- if (!HaveFieldsSubset(lambdaBody, arg, usedFields, *optCtx.ParentsMap)) {
- return node;
- }
-
- if (usedFields.size() == structType->GetSize()) {
- return node;
- }
-
- TVector<const TItemExprType*> subsetItems;
- for (const auto& item : structType->GetItems()) {
- if (usedFields.contains(item->GetName())) {
- subsetItems.push_back(item);
- }
- }
-
- auto subsetType = ctx.MakeType<TStructExprType>(subsetItems);
- YQL_CLOG(DEBUG, Core) << "FieldSubset for HoppingTraits";
- return Build<TCoHoppingTraits>(ctx, node->Pos())
- .ItemType(ExpandType(node->Pos(), *subsetType, ctx))
- .TimeExtractor(ctx.DeepCopyLambda(self.TimeExtractor().Ref()))
- .Hop(self.Hop())
- .Interval(self.Interval())
- .Delay(self.Delay())
- .DataWatermarks(self.DataWatermarks())
- .Done().Ptr();
- };
-
+ map[TCoHoppingTraits::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ TCoHoppingTraits self(node);
+
+ auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
+
+ const auto lambdaBody = self.TimeExtractor().Body().Ptr();
+ const auto& arg = self.TimeExtractor().Args().Arg(0).Ref();
+
+ TSet<TStringBuf> usedFields;
+ if (!HaveFieldsSubset(lambdaBody, arg, usedFields, *optCtx.ParentsMap)) {
+ return node;
+ }
+
+ if (usedFields.size() == structType->GetSize()) {
+ return node;
+ }
+
+ TVector<const TItemExprType*> subsetItems;
+ for (const auto& item : structType->GetItems()) {
+ if (usedFields.contains(item->GetName())) {
+ subsetItems.push_back(item);
+ }
+ }
+
+ auto subsetType = ctx.MakeType<TStructExprType>(subsetItems);
+ YQL_CLOG(DEBUG, Core) << "FieldSubset for HoppingTraits";
+ return Build<TCoHoppingTraits>(ctx, node->Pos())
+ .ItemType(ExpandType(node->Pos(), *subsetType, ctx))
+ .TimeExtractor(ctx.DeepCopyLambda(self.TimeExtractor().Ref()))
+ .Hop(self.Hop())
+ .Interval(self.Interval())
+ .Delay(self.Delay())
+ .DataWatermarks(self.DataWatermarks())
+ .Done().Ptr();
+ };
+
map["AggregationTraits"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
auto type = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Struct) {
diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
index b667a4f74f1..edfcaa15302 100644
--- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
+++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
@@ -552,12 +552,12 @@
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "HoppingTraits"},
"Children": [
- {"Index": 0, "Name": "ItemType", "Type": "TExprBase"},
- {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"},
- {"Index": 2, "Name": "Hop", "Type": "TExprBase"},
- {"Index": 3, "Name": "Interval", "Type": "TExprBase"},
- {"Index": 4, "Name": "Delay", "Type": "TExprBase"},
- {"Index": 5, "Name": "DataWatermarks", "Type": "TExprBase"}
+ {"Index": 0, "Name": "ItemType", "Type": "TExprBase"},
+ {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "Hop", "Type": "TExprBase"},
+ {"Index": 3, "Name": "Interval", "Type": "TExprBase"},
+ {"Index": 4, "Name": "Delay", "Type": "TExprBase"},
+ {"Index": 5, "Name": "DataWatermarks", "Type": "TExprBase"}
]
},
{
@@ -1601,38 +1601,38 @@
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "HoppingCore"},
"Children": [
- {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"},
- {"Index": 2, "Name": "Hop", "Type": "TExprBase"},
- {"Index": 3, "Name": "Interval", "Type": "TExprBase"},
- {"Index": 4, "Name": "Delay", "Type": "TExprBase"},
- {"Index": 5, "Name": "InitHandler", "Type": "TCoLambda"},
- {"Index": 6, "Name": "UpdateHandler", "Type": "TCoLambda"},
- {"Index": 7, "Name": "SaveHandler", "Type": "TExprBase"},
- {"Index": 8, "Name": "LoadHandler", "Type": "TExprBase"},
- {"Index": 9, "Name": "MergeHandler", "Type": "TCoLambda"},
- {"Index": 10, "Name": "FinishHandler", "Type": "TCoLambda"}
- ]
- },
- {
- "Name": "TCoMultiHoppingCore",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "MultiHoppingCore"},
- "Children": [
- {"Index": 1, "Name": "KeyExtractor", "Type": "TCoLambda"},
- {"Index": 2, "Name": "TimeExtractor", "Type": "TCoLambda"},
- {"Index": 3, "Name": "Hop", "Type": "TExprBase"},
- {"Index": 4, "Name": "Interval", "Type": "TExprBase"},
- {"Index": 5, "Name": "Delay", "Type": "TExprBase"},
- {"Index": 6, "Name": "DataWatermarks", "Type": "TExprBase"},
- {"Index": 7, "Name": "InitHandler", "Type": "TCoLambda"},
- {"Index": 8, "Name": "UpdateHandler", "Type": "TCoLambda"},
- {"Index": 9, "Name": "SaveHandler", "Type": "TExprBase"},
- {"Index": 10, "Name": "LoadHandler", "Type": "TExprBase"},
- {"Index": 11, "Name": "MergeHandler", "Type": "TCoLambda"},
- {"Index": 12, "Name": "FinishHandler", "Type": "TCoLambda"}
- ]
- },
- {
+ {"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "Hop", "Type": "TExprBase"},
+ {"Index": 3, "Name": "Interval", "Type": "TExprBase"},
+ {"Index": 4, "Name": "Delay", "Type": "TExprBase"},
+ {"Index": 5, "Name": "InitHandler", "Type": "TCoLambda"},
+ {"Index": 6, "Name": "UpdateHandler", "Type": "TCoLambda"},
+ {"Index": 7, "Name": "SaveHandler", "Type": "TExprBase"},
+ {"Index": 8, "Name": "LoadHandler", "Type": "TExprBase"},
+ {"Index": 9, "Name": "MergeHandler", "Type": "TCoLambda"},
+ {"Index": 10, "Name": "FinishHandler", "Type": "TCoLambda"}
+ ]
+ },
+ {
+ "Name": "TCoMultiHoppingCore",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "MultiHoppingCore"},
+ "Children": [
+ {"Index": 1, "Name": "KeyExtractor", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "TimeExtractor", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "Hop", "Type": "TExprBase"},
+ {"Index": 4, "Name": "Interval", "Type": "TExprBase"},
+ {"Index": 5, "Name": "Delay", "Type": "TExprBase"},
+ {"Index": 6, "Name": "DataWatermarks", "Type": "TExprBase"},
+ {"Index": 7, "Name": "InitHandler", "Type": "TCoLambda"},
+ {"Index": 8, "Name": "UpdateHandler", "Type": "TCoLambda"},
+ {"Index": 9, "Name": "SaveHandler", "Type": "TExprBase"},
+ {"Index": 10, "Name": "LoadHandler", "Type": "TExprBase"},
+ {"Index": 11, "Name": "MergeHandler", "Type": "TCoLambda"},
+ {"Index": 12, "Name": "FinishHandler", "Type": "TCoLambda"}
+ ]
+ },
+ {
"Name": "TCoCombineCore",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "CombineCore"},
diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp
index 269fccd5af3..5846e6cb108 100644
--- a/ydb/library/yql/core/type_ann/type_ann_core.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp
@@ -11545,15 +11545,15 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
break;
}
}
- for (auto& x : ctx.Types.DataSinks) {
- auto tokens = x->GetClusterTokens();
- auto token = tokens ? tokens->FindPtr(clusterName) : nullptr;
- if (token) {
- clusterCred.ConstructInPlace(TString(x->GetName()), "", *token);
- cred = clusterCred.Get();
- break;
- }
- }
+ for (auto& x : ctx.Types.DataSinks) {
+ auto tokens = x->GetClusterTokens();
+ auto token = tokens ? tokens->FindPtr(clusterName) : nullptr;
+ if (token) {
+ clusterCred.ConstructInPlace(TString(x->GetName()), "", *token);
+ cred = clusterCred.Get();
+ break;
+ }
+ }
}
}
}
@@ -12996,7 +12996,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["GroupingCore"] = &GroupingCoreWrapper;
Functions["HoppingTraits"] = &HoppingTraitsWrapper;
Functions["HoppingCore"] = &HoppingCoreWrapper;
- Functions["MultiHoppingCore"] = &MultiHoppingCoreWrapper;
+ Functions["MultiHoppingCore"] = &MultiHoppingCoreWrapper;
Functions["EquiJoin"] = &EquiJoinWrapper;
Functions["OptionalReduce"] = &OptionalReduceWrapper;
Functions["OptionalItemType"] = &TypeArgWrapper<ETypeArgument::OptionalItem>;
diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp
index ac56e2c71a8..b145cde3893 100644
--- a/ydb/library/yql/core/type_ann/type_ann_list.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp
@@ -4546,7 +4546,7 @@ namespace {
const auto item = tupleType->GetItems()[index];
rowColumns.push_back(ctx.Expr.MakeType<TItemExprType>(
child->Head().Child(index)->Content(),
- (isOptional || (input->Child(1)->ChildrenSize() == 0 && !isHopping)) &&
+ (isOptional || (input->Child(1)->ChildrenSize() == 0 && !isHopping)) &&
item->GetKind() != ETypeAnnotationKind::Optional ? ctx.Expr.MakeType<TOptionalExprType>(item) : item));
}
} else {
@@ -5399,7 +5399,7 @@ namespace {
IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
- if (!EnsureMinArgsCount(*input, 5, ctx.Expr) || !EnsureMaxArgsCount(*input, 6, ctx.Expr)) {
+ if (!EnsureMinArgsCount(*input, 5, ctx.Expr) || !EnsureMaxArgsCount(*input, 6, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) {
@@ -5470,46 +5470,46 @@ namespace {
return convertStatus;
}
- if (input->ChildrenSize() == 6) {
- const auto& dataWatermarksNodePtr = input->ChildRef(5);
- if (dataWatermarksNodePtr->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Unit) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(dataWatermarksNodePtr->Pos()), TStringBuilder()
- << "Expected unit type, but got: "
- << *dataWatermarksNodePtr->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
- }
-
+ if (input->ChildrenSize() == 6) {
+ const auto& dataWatermarksNodePtr = input->ChildRef(5);
+ if (dataWatermarksNodePtr->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Unit) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(dataWatermarksNodePtr->Pos()), TStringBuilder()
+ << "Expected unit type, but got: "
+ << *dataWatermarksNodePtr->GetTypeAnn()));
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
- if (!EnsureArgsCount(*input, 11, ctx.Expr)) {
+ if (!EnsureArgsCount(*input, 11, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
-
+
auto& item = input->ChildRef(0);
- auto& lambdaTimeExtractor = input->ChildRef(1);
- auto* hop = input->Child(2);
- auto* interval = input->Child(3);
- auto* delay = input->Child(4);
-
- auto& lambdaInit = input->ChildRef(5);
- auto& lambdaUpdate = input->ChildRef(6);
- auto& saveLambda = input->ChildRef(7);
- auto& loadLambda = input->ChildRef(8);
- auto& lambdaMerge = input->ChildRef(9);
- auto& lambdaFinish = input->ChildRef(10);
-
+ auto& lambdaTimeExtractor = input->ChildRef(1);
+ auto* hop = input->Child(2);
+ auto* interval = input->Child(3);
+ auto* delay = input->Child(4);
+
+ auto& lambdaInit = input->ChildRef(5);
+ auto& lambdaUpdate = input->ChildRef(6);
+ auto& saveLambda = input->ChildRef(7);
+ auto& loadLambda = input->ChildRef(8);
+ auto& lambdaMerge = input->ChildRef(9);
+ auto& lambdaFinish = input->ChildRef(10);
+
if (!EnsureStreamType(*item, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1);
- status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1));
+ auto status = ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1);
+ status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1));
status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2));
status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2));
status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 2));
@@ -5524,24 +5524,24 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr))
- {
- return IGraphTransformer::TStatus::Error;
- }
- if (!lambdaTimeExtractor->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- if (!EnsureSpecificDataType(*hop, EDataSlot::Interval, ctx.Expr, true)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!EnsureSpecificDataType(*interval, EDataSlot::Interval, ctx.Expr, true)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!EnsureSpecificDataType(*delay, EDataSlot::Interval, ctx.Expr, true)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr))
+ {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!lambdaTimeExtractor->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ if (!EnsureSpecificDataType(*hop, EDataSlot::Interval, ctx.Expr, true)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!EnsureSpecificDataType(*interval, EDataSlot::Interval, ctx.Expr, true)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!EnsureSpecificDataType(*delay, EDataSlot::Interval, ctx.Expr, true)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -5635,167 +5635,167 @@ namespace {
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
- if (!EnsureArgsCount(*input, 13, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- auto& item = input->ChildRef(0);
- auto& lambdaKeyExtractor = input->ChildRef(1);
-
- auto& lambdaTimeExtractor = input->ChildRef(2);
- auto* hop = input->Child(3);
- auto* interval = input->Child(4);
- auto* delay = input->Child(5);
-
- auto& lambdaInit = input->ChildRef(7);
- auto& lambdaUpdate = input->ChildRef(8);
- auto& saveLambda = input->ChildRef(9);
- auto& loadLambda = input->ChildRef(10);
- auto& lambdaMerge = input->ChildRef(11);
- auto& lambdaFinish = input->ChildRef(12);
-
- if (!EnsureStreamType(*item, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- auto status = ConvertToLambda(lambdaKeyExtractor, ctx.Expr, 1);
- status = status.Combine(ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1));
- status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1));
- status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2));
- status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2));
- status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 3));
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- auto timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp));
- auto itemType = item->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
-
- if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaKeyExtractor, {itemType}, ctx.Expr))
- {
- return IGraphTransformer::TStatus::Error;
- }
- auto keyType = lambdaKeyExtractor->GetTypeAnn();
- if (!keyType) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr))
- {
- return IGraphTransformer::TStatus::Error;
- }
- if (!lambdaTimeExtractor->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- if (!EnsureSpecificDataType(*hop, EDataSlot::Interval, ctx.Expr, true)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!EnsureSpecificDataType(*interval, EDataSlot::Interval, ctx.Expr, true)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!EnsureSpecificDataType(*delay, EDataSlot::Interval, ctx.Expr, true)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!lambdaInit->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- auto stateType = lambdaInit->GetTypeAnn();
- if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!lambdaUpdate->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: "
- << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!lambdaMerge->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: "
- << *stateType << ", but got: " << *lambdaMerge->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {keyType, stateType, timeType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!lambdaFinish->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
- if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() <<
- "Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() <<
- "Save and load lambdas must be specified at the same time"));
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!saveLambda->IsCallable("Void")) {
- auto status = ConvertToLambda(saveLambda, ctx.Expr, 1);
- status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1));
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!saveLambda->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- auto savedStateType = saveLambda->GetTypeAnn();
- if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!loadLambda->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, "
- << *loadLambda->GetTypeAnn() << " != " << *stateType));
- return IGraphTransformer::TStatus::Error;
- }
- }
-
- input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(lambdaFinish->GetTypeAnn()));
- return IGraphTransformer::TStatus::Ok;
- }
-
+ IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
+ if (!EnsureArgsCount(*input, 13, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ auto& item = input->ChildRef(0);
+ auto& lambdaKeyExtractor = input->ChildRef(1);
+
+ auto& lambdaTimeExtractor = input->ChildRef(2);
+ auto* hop = input->Child(3);
+ auto* interval = input->Child(4);
+ auto* delay = input->Child(5);
+
+ auto& lambdaInit = input->ChildRef(7);
+ auto& lambdaUpdate = input->ChildRef(8);
+ auto& saveLambda = input->ChildRef(9);
+ auto& loadLambda = input->ChildRef(10);
+ auto& lambdaMerge = input->ChildRef(11);
+ auto& lambdaFinish = input->ChildRef(12);
+
+ if (!EnsureStreamType(*item, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ auto status = ConvertToLambda(lambdaKeyExtractor, ctx.Expr, 1);
+ status = status.Combine(ConvertToLambda(lambdaTimeExtractor, ctx.Expr, 1));
+ status = status.Combine(ConvertToLambda(lambdaInit, ctx.Expr, 1));
+ status = status.Combine(ConvertToLambda(lambdaUpdate, ctx.Expr, 2));
+ status = status.Combine(ConvertToLambda(lambdaMerge, ctx.Expr, 2));
+ status = status.Combine(ConvertToLambda(lambdaFinish, ctx.Expr, 3));
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ auto timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp));
+ auto itemType = item->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
+
+ if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaKeyExtractor, {itemType}, ctx.Expr))
+ {
+ return IGraphTransformer::TStatus::Error;
+ }
+ auto keyType = lambdaKeyExtractor->GetTypeAnn();
+ if (!keyType) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr))
+ {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!lambdaTimeExtractor->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ if (!EnsureSpecificDataType(*hop, EDataSlot::Interval, ctx.Expr, true)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!EnsureSpecificDataType(*interval, EDataSlot::Interval, ctx.Expr, true)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!EnsureSpecificDataType(*delay, EDataSlot::Interval, ctx.Expr, true)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!lambdaInit->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ auto stateType = lambdaInit->GetTypeAnn();
+ if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!lambdaUpdate->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: "
+ << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!lambdaMerge->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: "
+ << *stateType << ", but got: " << *lambdaMerge->GetTypeAnn()));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {keyType, stateType, timeType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!lambdaFinish->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() <<
+ "Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn()));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (saveLambda->IsCallable("Void") != loadLambda->IsCallable("Void")) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(saveLambda->Pos()), TStringBuilder() <<
+ "Save and load lambdas must be specified at the same time"));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!saveLambda->IsCallable("Void")) {
+ auto status = ConvertToLambda(saveLambda, ctx.Expr, 1);
+ status = status.Combine(ConvertToLambda(loadLambda, ctx.Expr, 1));
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!saveLambda->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ auto savedStateType = saveLambda->GetTypeAnn();
+ if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!loadLambda->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, "
+ << *loadLambda->GetTypeAnn() << " != " << *stateType));
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
+ input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(lambdaFinish->GetTypeAnn()));
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus CombineCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
if (!EnsureMinMaxArgsCount(*input, 5U, 6U, ctx.Expr)) {
diff --git a/ydb/library/yql/core/type_ann/type_ann_list.h b/ydb/library/yql/core/type_ann/type_ann_list.h
index 681400e1b4d..608b813fe8b 100644
--- a/ydb/library/yql/core/type_ann/type_ann_list.h
+++ b/ydb/library/yql/core/type_ann/type_ann_list.h
@@ -105,7 +105,7 @@ namespace NTypeAnnImpl {
IGraphTransformer::TStatus WinRowNumberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus WinRankWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus HoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
- IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
+ IGraphTransformer::TStatus MultiHoppingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus HoppingTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus ListMinWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus ListMaxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp
index 673b1a1be35..f2b793af8d3 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp
@@ -2144,18 +2144,18 @@ IGraphTransformer::TStatus ConvertToLambda(TExprNode::TPtr& node, TExprContext&
return IGraphTransformer::TStatus::Repeat;
}
-bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional) {
+bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional) {
if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got lambda"));
return false;
}
- if (allowOptional && node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
- auto optionalType = node.GetTypeAnn()->Cast<TOptionalExprType>();
- return EnsureSpecificDataType(node.Pos(), *optionalType->GetItemType(), expectedDataSlot, ctx);
- }
-
+ if (allowOptional && node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+ auto optionalType = node.GetTypeAnn()->Cast<TOptionalExprType>();
+ return EnsureSpecificDataType(node.Pos(), *optionalType->GetItemType(), expectedDataSlot, ctx);
+ }
+
if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got: " << *node.GetTypeAnn()));
return false;
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.h b/ydb/library/yql/core/yql_expr_type_annotation.h
index f81802eae3d..a865aa27cd9 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.h
+++ b/ydb/library/yql/core/yql_expr_type_annotation.h
@@ -82,7 +82,7 @@ bool EnsureVariantType(const TExprNode& node, TExprContext& ctx);
bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx);
bool EnsureDataType(const TExprNode& node, TExprContext& ctx);
bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx);
-bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional = false);
+bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional = false);
bool EnsureSpecificDataType(TPositionHandle position, const TTypeAnnotationNode& type, EDataSlot expectedDataSlot, TExprContext& ctx);
bool EnsureStringOrUtf8Type(const TExprNode& node, TExprContext& ctx);
bool EnsureStringOrUtf8Type(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx);
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp
index 131f758d552..1a1611ec81e 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp
@@ -498,11 +498,11 @@ void TDqComputeActorChannels::PassAway() {
IActor::PassAway();
}
-void TDqComputeActorChannels::SetInputChannelPeer(ui64 channelId, const TActorId& peer) {
- TInputChannelState& inputChannel = InCh(channelId);
- inputChannel.Peer = peer;
-}
-
+void TDqComputeActorChannels::SetInputChannelPeer(ui64 channelId, const TActorId& peer) {
+ TInputChannelState& inputChannel = InCh(channelId);
+ inputChannel.Peer = peer;
+}
+
void TDqComputeActorChannels::SetOutputChannelPeer(ui64 channelId, const TActorId& peer) {
TOutputChannelState& outputChannel = OutCh(channelId);
outputChannel.Peer = peer;
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h
index 9b243ef54b5..648b1d8b563 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h
@@ -67,7 +67,7 @@ private:
public:
void SetCheckpointsSupport(); // Finished channels will be polled for checkpoints.
- void SetInputChannelPeer(ui64 channelId, const NActors::TActorId& peer);
+ void SetInputChannelPeer(ui64 channelId, const NActors::TActorId& peer);
void SetOutputChannelPeer(ui64 channelId, const NActors::TActorId& peer);
bool CanSendChannelData(ui64 channelId);
void SendChannelData(NDqProto::TChannelData&& channelData);
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
index 713afb7d54a..deaadefb42c 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
@@ -5,7 +5,7 @@
#include "dq_compute_actor_checkpoints.h"
#include "dq_compute_actor_sinks.h"
#include "dq_compute_actor_sources.h"
-#include "dq_compute_issues_buffer.h"
+#include "dq_compute_issues_buffer.h"
#include <ydb/core/scheme/scheme_tabledefs.h> // TODO: TTableId
#include <ydb/core/base/kikimr_issue.h>
@@ -54,7 +54,7 @@ enum : ui64 {
ComputeActorCurrentStateVersion = 2,
};
-constexpr ui32 IssuesBufferSize = 16;
+constexpr ui32 IssuesBufferSize = 16;
template<typename TDerived>
class TDqComputeActorBase : public NActors::TActorBootstrapped<TDerived>
@@ -257,7 +257,7 @@ protected:
virtual void DoExecuteImpl() {
auto sourcesState = GetSourcesState();
- PollSourceActors();
+ PollSourceActors();
ERunStatus status = TaskRunner->Run();
CA_LOG_D("Resume execution, run status: " << status);
@@ -338,9 +338,9 @@ protected:
if (status == ERunStatus::PendingInput && ProcessOutputsState.AllOutputsFinished) {
CA_LOG_D("All outputs have been finished. Consider finished");
- status = ERunStatus::Finished;
- }
-
+ status = ERunStatus::Finished;
+ }
+
if (status != ERunStatus::Finished) {
// If the incoming channel's buffer was full at the moment when last ChannelDataAck event had been sent,
// there will be no attempts to send a new piece of data from the other side of this channel.
@@ -670,7 +670,7 @@ protected:
struct TInputChannelInfo {
ui64 ChannelId;
IDqInputChannel::TPtr Channel;
- bool HasPeer = false;
+ bool HasPeer = false;
std::optional<NDqProto::TCheckpoint> PendingCheckpoint;
const NDqProto::ECheckpointingMode CheckpointingMode;
ui64 FreeSpace = 0;
@@ -703,11 +703,11 @@ protected:
IDqSource::TPtr Source;
IDqSourceActor* SourceActor = nullptr;
NActors::IActor* Actor = nullptr;
- TIssuesBuffer IssuesBuffer;
+ TIssuesBuffer IssuesBuffer;
bool Finished = false;
i64 FreeSpace = 1;
bool PushStarted = false;
-
+
TSourceInfo(ui64 index) : Index(index), IssuesBuffer(IssuesBufferSize) {}
};
@@ -734,11 +734,11 @@ protected:
IDqSinkActor* SinkActor = nullptr;
NActors::IActor* Actor = nullptr;
bool Finished = false; // If sink is in finished state, it receives only checkpoints.
- TIssuesBuffer IssuesBuffer;
+ TIssuesBuffer IssuesBuffer;
bool PopStarted = false;
i64 SinkActorFreeSpaceBeforeSend = 0;
-
- TSinkInfo() : IssuesBuffer(IssuesBufferSize) {}
+
+ TSinkInfo() : IssuesBuffer(IssuesBufferSize) {}
};
protected:
@@ -796,31 +796,31 @@ protected:
CA_LOG_D("Received channels info: " << record.ShortDebugString());
for (auto& channelUpdate : record.GetUpdate()) {
- TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelUpdate.GetId());
- if (inputChannel && !inputChannel->HasPeer && channelUpdate.GetSrcEndpoint().HasActorId()) {
+ TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelUpdate.GetId());
+ if (inputChannel && !inputChannel->HasPeer && channelUpdate.GetSrcEndpoint().HasActorId()) {
auto peer = NActors::ActorIdFromProto(channelUpdate.GetSrcEndpoint().GetActorId());
-
- CA_LOG_D("Update input channelId: " << channelUpdate.GetId() << ", peer: " << peer);
-
- Channels->SetInputChannelPeer(channelUpdate.GetId(), peer);
- inputChannel->HasPeer = true;
-
- continue;
- }
-
+
+ CA_LOG_D("Update input channelId: " << channelUpdate.GetId() << ", peer: " << peer);
+
+ Channels->SetInputChannelPeer(channelUpdate.GetId(), peer);
+ inputChannel->HasPeer = true;
+
+ continue;
+ }
+
TOutputChannelInfo* outputChannel = OutputChannelsMap.FindPtr(channelUpdate.GetId());
- if (outputChannel && !outputChannel->HasPeer && channelUpdate.GetDstEndpoint().HasActorId()) {
+ if (outputChannel && !outputChannel->HasPeer && channelUpdate.GetDstEndpoint().HasActorId()) {
auto peer = NActors::ActorIdFromProto(channelUpdate.GetDstEndpoint().GetActorId());
CA_LOG_D("Update output channelId: " << channelUpdate.GetId() << ", peer: " << peer);
Channels->SetOutputChannelPeer(channelUpdate.GetId(), peer);
outputChannel->HasPeer = true;
-
- continue;
+
+ continue;
}
-
- YQL_ENSURE(inputChannel || outputChannel, "Unknown channelId: " << channelUpdate.GetId() << ", task: " << Task.GetId());
+
+ YQL_ENSURE(inputChannel || outputChannel, "Unknown channelId: " << channelUpdate.GetId() << ", task: " << Task.GetId());
}
DoExecute();
@@ -1171,16 +1171,16 @@ protected:
const ui64 i = inputIndex; // Crutch for clang
CA_LOG_D("Create source actor for input " << i << " " << inputDesc);
std::tie(source.SourceActor, source.Actor) = SourceActorFactory->CreateDqSourceActor(
- IDqSourceActorFactory::TArguments{
- .InputDesc = inputDesc,
- .InputIndex = inputIndex,
- .TxId = TxId,
+ IDqSourceActorFactory::TArguments{
+ .InputDesc = inputDesc,
+ .InputIndex = inputIndex,
+ .TxId = TxId,
.SecureParams = secureParams,
.TaskParams = taskParams,
- .Callback = this,
+ .Callback = this,
.TypeEnv = typeEnv,
.HolderFactory = holderFactory
- });
+ });
this->RegisterWithSameMailbox(source.Actor);
}
if (TaskRunner) {
@@ -1195,15 +1195,15 @@ protected:
const ui64 i = outputIndex; // Crutch for clang
CA_LOG_D("Create sink actor for output " << i << " " << outputDesc);
std::tie(sink.SinkActor, sink.Actor) = SinkActorFactory->CreateDqSinkActor(
- IDqSinkActorFactory::TArguments {
- .OutputDesc = outputDesc,
- .OutputIndex = outputIndex,
- .TxId = TxId,
+ IDqSinkActorFactory::TArguments {
+ .OutputDesc = outputDesc,
+ .OutputIndex = outputIndex,
+ .TxId = TxId,
.SecureParams = secureParams,
- .Callback = this,
+ .Callback = this,
.TypeEnv = typeEnv,
.HolderFactory = holderFactory
- });
+ });
this->RegisterWithSameMailbox(sink.Actor);
}
}
@@ -1242,23 +1242,23 @@ protected:
ContinueExecute();
}
- void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) override {
- if (!isFatal) {
- SourcesMap.at(inputIndex).IssuesBuffer.Push(issues);
- return;
- }
-
+ void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) override {
+ if (!isFatal) {
+ SourcesMap.at(inputIndex).IssuesBuffer.Push(issues);
+ return;
+ }
+
TString desc = issues.ToString();
CA_LOG_E("Source[" << inputIndex << "] fatal error: " << desc);
InternalError(TIssuesIds::DEFAULT_ERROR, desc);
}
- void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) override {
- if (!isFatal) {
- SinksMap.at(outputIndex).IssuesBuffer.Push(issues);
- return;
- }
-
+ void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) override {
+ if (!isFatal) {
+ SinksMap.at(outputIndex).IssuesBuffer.Push(issues);
+ return;
+ }
+
TString desc = issues.ToString();
CA_LOG_E("Sink[" << outputIndex << "] fatal error: " << desc);
InternalError(TIssuesIds::DEFAULT_ERROR, desc);
@@ -1413,13 +1413,13 @@ private:
}
}
}
-
+
static_cast<TDerived*>(this)->FillExtraStats(dst, last);
-
+
if (last) {
BasicStats.reset();
ProfileStats.reset();
- }
+ }
}
void ReportStats(TInstant now) {
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp
index 2176d57488a..5ae40ef1ab6 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.cpp
@@ -3,16 +3,16 @@
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h>
#include <ydb/library/yql/dq/common/dq_common.h>
-
+
namespace NYql::NDq {
-std::pair<IDqSourceActor*, NActors::IActor*> TDqSourceFactory::CreateDqSourceActor(IDqSourceActorFactory::TArguments&& args) const
+std::pair<IDqSourceActor*, NActors::IActor*> TDqSourceFactory::CreateDqSourceActor(IDqSourceActorFactory::TArguments&& args) const
{
- const TString& type = args.InputDesc.GetSource().GetType();
+ const TString& type = args.InputDesc.GetSource().GetType();
YQL_ENSURE(!type.empty(), "Attempt to create source actor of empty type");
const TCreatorFunction* creatorFunc = CreatorsByType.FindPtr(type);
YQL_ENSURE(creatorFunc, "Unknown type of source actor: \"" << type << "\"");
- std::pair<IDqSourceActor*, NActors::IActor*> actor = (*creatorFunc)(std::move(args));
+ std::pair<IDqSourceActor*, NActors::IActor*> actor = (*creatorFunc)(std::move(args));
Y_VERIFY(actor.first);
Y_VERIFY(actor.second);
return actor;
@@ -24,13 +24,13 @@ void TDqSourceFactory::Register(const TString& type, TCreatorFunction creator)
Y_VERIFY(registered);
}
-std::pair<IDqSinkActor*, NActors::IActor*> TDqSinkFactory::CreateDqSinkActor(IDqSinkActorFactory::TArguments&& args) const
+std::pair<IDqSinkActor*, NActors::IActor*> TDqSinkFactory::CreateDqSinkActor(IDqSinkActorFactory::TArguments&& args) const
{
- const TString& type = args.OutputDesc.GetSink().GetType();
+ const TString& type = args.OutputDesc.GetSink().GetType();
YQL_ENSURE(!type.empty(), "Attempt to create sink actor of empty type");
const TCreatorFunction* creatorFunc = CreatorsByType.FindPtr(type);
YQL_ENSURE(creatorFunc, "Unknown type of sink actor: \"" << type << "\"");
- std::pair<IDqSinkActor*, NActors::IActor*> actor = (*creatorFunc)(std::move(args));
+ std::pair<IDqSinkActor*, NActors::IActor*> actor = (*creatorFunc)(std::move(args));
Y_VERIFY(actor.first);
Y_VERIFY(actor.second);
return actor;
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h
index 7b31e818a86..db57796f5d4 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h
@@ -4,7 +4,7 @@
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/dq/common/dq_common.h>
-
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/utils/yql_panic.h>
@@ -17,30 +17,30 @@ concept TCastsToSourceActorPair =
std::is_convertible_v<T, std::pair<IDqSourceActor*, NActors::IActor*>>;
template <class T, class TProto>
-concept TSourceActorCreatorFunc = requires(T f, TProto&& settings, IDqSourceActorFactory::TArguments args) {
- { f(std::move(settings), std::move(args)) } -> TCastsToSourceActorPair;
+concept TSourceActorCreatorFunc = requires(T f, TProto&& settings, IDqSourceActorFactory::TArguments args) {
+ { f(std::move(settings), std::move(args)) } -> TCastsToSourceActorPair;
};
class TDqSourceFactory : public IDqSourceActorFactory {
public:
- using TCreatorFunction = std::function<std::pair<IDqSourceActor*, NActors::IActor*>(TArguments&& args)>;
+ using TCreatorFunction = std::function<std::pair<IDqSourceActor*, NActors::IActor*>(TArguments&& args)>;
- std::pair<IDqSourceActor*, NActors::IActor*> CreateDqSourceActor(TArguments&& args) const override;
+ std::pair<IDqSourceActor*, NActors::IActor*> CreateDqSourceActor(TArguments&& args) const override;
void Register(const TString& type, TCreatorFunction creator);
template <class TProtoMsg, TSourceActorCreatorFunc<TProtoMsg> TCreatorFunc>
void Register(const TString& type, TCreatorFunc creator) {
Register(type,
- [creator = std::move(creator), type](TArguments&& args)
+ [creator = std::move(creator), type](TArguments&& args)
{
- const google::protobuf::Any& settingsAny = args.InputDesc.GetSource().GetSettings();
+ const google::protobuf::Any& settingsAny = args.InputDesc.GetSource().GetSettings();
YQL_ENSURE(settingsAny.Is<TProtoMsg>(),
"Source \"" << type << "\" settings are expected to have protobuf type " << TProtoMsg::descriptor()->full_name()
<< ", but got " << settingsAny.type_url());
TProtoMsg settings;
YQL_ENSURE(settingsAny.UnpackTo(&settings), "Failed to unpack settings of type \"" << type << "\"");
- return creator(std::move(settings), std::move(args));
+ return creator(std::move(settings), std::move(args));
});
}
@@ -53,30 +53,30 @@ concept TCastsToSinkActorPair =
std::is_convertible_v<T, std::pair<IDqSinkActor*, NActors::IActor*>>;
template <class T, class TProto>
-concept TSinkActorCreatorFunc = requires(T f, TProto&& settings, IDqSinkActorFactory::TArguments&& args) {
- { f(std::move(settings), std::move(args)) } -> TCastsToSinkActorPair;
+concept TSinkActorCreatorFunc = requires(T f, TProto&& settings, IDqSinkActorFactory::TArguments&& args) {
+ { f(std::move(settings), std::move(args)) } -> TCastsToSinkActorPair;
};
class TDqSinkFactory : public IDqSinkActorFactory {
public:
- using TCreatorFunction = std::function<std::pair<IDqSinkActor*, NActors::IActor*>(TArguments&& args)>;
+ using TCreatorFunction = std::function<std::pair<IDqSinkActor*, NActors::IActor*>(TArguments&& args)>;
- std::pair<IDqSinkActor*, NActors::IActor*> CreateDqSinkActor(TArguments&& args) const override;
+ std::pair<IDqSinkActor*, NActors::IActor*> CreateDqSinkActor(TArguments&& args) const override;
void Register(const TString& type, TCreatorFunction creator);
template <class TProtoMsg, TSinkActorCreatorFunc<TProtoMsg> TCreatorFunc>
void Register(const TString& type, TCreatorFunc creator) {
Register(type,
- [creator = std::move(creator), type](TArguments&& args)
+ [creator = std::move(creator), type](TArguments&& args)
{
- const google::protobuf::Any& settingsAny = args.OutputDesc.GetSink().GetSettings();
+ const google::protobuf::Any& settingsAny = args.OutputDesc.GetSink().GetSettings();
YQL_ENSURE(settingsAny.Is<TProtoMsg>(),
"Sink \"" << type << "\" settings are expected to have protobuf type " << TProtoMsg::descriptor()->full_name()
<< ", but got " << settingsAny.type_url());
TProtoMsg settings;
YQL_ENSURE(settingsAny.UnpackTo(&settings), "Failed to unpack settings of type \"" << type << "\"");
- return creator(std::move(settings), std::move(args));
+ return creator(std::move(settings), std::move(args));
});
}
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h
index e3e74da728e..5c2db2ac8d8 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h
@@ -10,7 +10,7 @@
namespace NYql::NDqProto {
class TCheckpoint;
-class TTaskOutput;
+class TTaskOutput;
class TSinkState;
} // namespace NYql::NDqProto
@@ -42,7 +42,7 @@ namespace NYql::NDq {
struct IDqSinkActor {
struct ICallbacks { // Compute actor
virtual void ResumeExecution() = 0;
- virtual void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) = 0;
+ virtual void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) = 0;
// Checkpointing
virtual void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint& checkpoint) = 0;
@@ -74,20 +74,20 @@ struct IDqSinkActor {
struct IDqSinkActorFactory : public TThrRefBase {
using TPtr = TIntrusivePtr<IDqSinkActorFactory>;
- struct TArguments {
- const NDqProto::TTaskOutput& OutputDesc;
- ui64 OutputIndex;
- TTxId TxId;
- const THashMap<TString, TString>& SecureParams;
- IDqSinkActor::ICallbacks* Callback;
- const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv;
- const NKikimr::NMiniKQL::THolderFactory& HolderFactory;
- };
-
+ struct TArguments {
+ const NDqProto::TTaskOutput& OutputDesc;
+ ui64 OutputIndex;
+ TTxId TxId;
+ const THashMap<TString, TString>& SecureParams;
+ IDqSinkActor::ICallbacks* Callback;
+ const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv;
+ const NKikimr::NMiniKQL::THolderFactory& HolderFactory;
+ };
+
// Creates sink actor.
// Could throw YQL errors.
// IActor* and IDqSinkActor* returned by method must point to the objects with consistent lifetime.
- virtual std::pair<IDqSinkActor*, NActors::IActor*> CreateDqSinkActor(TArguments&& args) const = 0;
+ virtual std::pair<IDqSinkActor*, NActors::IActor*> CreateDqSinkActor(TArguments&& args) const = 0;
};
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h
index 647e0645ca4..d2a646cab62 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h
@@ -38,7 +38,7 @@ namespace NYql::NDq {
struct IDqSourceActor {
struct ICallbacks {
virtual void OnNewSourceDataArrived(ui64 inputIndex) = 0;
- virtual void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) = 0;
+ virtual void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) = 0;
virtual ~ICallbacks() = default;
};
@@ -63,21 +63,21 @@ struct IDqSourceActor {
struct IDqSourceActorFactory : public TThrRefBase {
using TPtr = TIntrusivePtr<IDqSourceActorFactory>;
- struct TArguments {
- const NDqProto::TTaskInput& InputDesc;
- ui64 InputIndex;
- TTxId TxId;
- const THashMap<TString, TString>& SecureParams;
- const THashMap<TString, TString>& TaskParams;
- IDqSourceActor::ICallbacks* Callback;
- const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv;
- const NKikimr::NMiniKQL::THolderFactory& HolderFactory;
- };
-
+ struct TArguments {
+ const NDqProto::TTaskInput& InputDesc;
+ ui64 InputIndex;
+ TTxId TxId;
+ const THashMap<TString, TString>& SecureParams;
+ const THashMap<TString, TString>& TaskParams;
+ IDqSourceActor::ICallbacks* Callback;
+ const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv;
+ const NKikimr::NMiniKQL::THolderFactory& HolderFactory;
+ };
+
// Creates source actor.
// Could throw YQL errors.
// IActor* and IDqSourceActor* returned by method must point to the objects with consistent lifetime.
- virtual std::pair<IDqSourceActor*, NActors::IActor*> CreateDqSourceActor(TArguments&& args) const = 0;
+ virtual std::pair<IDqSourceActor*, NActors::IActor*> CreateDqSourceActor(TArguments&& args) const = 0;
};
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp
index 64377cbb8a1..cc13ce4d272 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_stats.cpp
@@ -35,13 +35,13 @@ void FillTaskRunnerStats(ui64 taskId, ui32 stageId, const TDqTaskRunnerStats& ta
protoBucket->SetValue(snapshot->Value(i));
}
}
-
- for (const auto& stat : taskStats.MkqlStats) {
- auto* s = protoTask->MutableMkqlStats()->Add();
- s->SetName(TString(stat.Key.GetName()));
- s->SetValue(stat.Value);
- s->SetDeriv(stat.Key.IsDeriv());
- }
+
+ for (const auto& stat : taskStats.MkqlStats) {
+ auto* s = protoTask->MutableMkqlStats()->Add();
+ s->SetName(TString(stat.Key.GetName()));
+ s->SetValue(stat.Value);
+ s->SetDeriv(stat.Key.IsDeriv());
+ }
}
ui64 firstRowTs = std::numeric_limits<ui64>::max();
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp
index 5d740a0a873..310fb96f91b 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.cpp
@@ -1,50 +1,50 @@
-#include "dq_compute_issues_buffer.h"
-
-#include <algorithm>
-
-namespace NYql {
-namespace NDq {
-
-TIssuesBuffer::TIssuesBuffer(ui32 capacity)
- : Capacity(capacity)
- , Issues(capacity)
-{
- Y_ENSURE(Capacity);
-}
-
-ui32 TIssuesBuffer::GetAllAddedIssuesCount() const {
- return AllAddedIssuesCount;
-}
-
-void TIssuesBuffer::Push(const TIssues& issues) {
- AllAddedIssuesCount++;
-
- auto& issueInfo = Issues.at((HeadIndex + ItemsCount) % Capacity);
- issueInfo.Issues = issues;
- issueInfo.OccuredAt = TInstant::Now();
-
- if (ItemsCount < Capacity) {
- ItemsCount++;
- return;
- }
-
- HeadIndex = (HeadIndex + 1) % Capacity;
-}
-
-std::vector<TIssueInfo> TIssuesBuffer::Dump() {
- std::rotate(Issues.begin(), std::next(Issues.begin(), HeadIndex), Issues.end());
- Issues.resize(ItemsCount);
- auto res = std::move(Issues);
- Clear();
- return res;
-}
-
-void TIssuesBuffer::Clear() {
- Issues.clear();
- Issues.resize(Capacity);
- HeadIndex = 0;
- ItemsCount = 0;
-}
-
-}
-}
+#include "dq_compute_issues_buffer.h"
+
+#include <algorithm>
+
+namespace NYql {
+namespace NDq {
+
+TIssuesBuffer::TIssuesBuffer(ui32 capacity)
+ : Capacity(capacity)
+ , Issues(capacity)
+{
+ Y_ENSURE(Capacity);
+}
+
+ui32 TIssuesBuffer::GetAllAddedIssuesCount() const {
+ return AllAddedIssuesCount;
+}
+
+void TIssuesBuffer::Push(const TIssues& issues) {
+ AllAddedIssuesCount++;
+
+ auto& issueInfo = Issues.at((HeadIndex + ItemsCount) % Capacity);
+ issueInfo.Issues = issues;
+ issueInfo.OccuredAt = TInstant::Now();
+
+ if (ItemsCount < Capacity) {
+ ItemsCount++;
+ return;
+ }
+
+ HeadIndex = (HeadIndex + 1) % Capacity;
+}
+
+std::vector<TIssueInfo> TIssuesBuffer::Dump() {
+ std::rotate(Issues.begin(), std::next(Issues.begin(), HeadIndex), Issues.end());
+ Issues.resize(ItemsCount);
+ auto res = std::move(Issues);
+ Clear();
+ return res;
+}
+
+void TIssuesBuffer::Clear() {
+ Issues.clear();
+ Issues.resize(Capacity);
+ HeadIndex = 0;
+ ItemsCount = 0;
+}
+
+}
+}
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h b/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h
index a962f61e8af..9bf8f4bf497 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h
@@ -1,39 +1,39 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/public/issue/yql_issue.h>
-
-#include <util/datetime/base.h>
-
-#include <vector>
-
-namespace NYql {
-namespace NDq {
-
-struct TIssueInfo {
- TIssues Issues;
- TInstant OccuredAt;
-};
-
-// Simple circular buffer
-struct TIssuesBuffer {
-public:
- explicit TIssuesBuffer(ui32 capacity);
-
- ui32 GetAllAddedIssuesCount() const;
- void Push(const TIssues& issues);
- std::vector<TIssueInfo> Dump();
-
-private:
- void Clear();
-
-private:
- ui32 Capacity;
- ui32 AllAddedIssuesCount = 0;
-
- std::vector<TIssueInfo> Issues;
- ui32 HeadIndex = 0;
- ui32 ItemsCount = 0;
-};
-
-}
-}
+
+#include <util/datetime/base.h>
+
+#include <vector>
+
+namespace NYql {
+namespace NDq {
+
+struct TIssueInfo {
+ TIssues Issues;
+ TInstant OccuredAt;
+};
+
+// Simple circular buffer
+struct TIssuesBuffer {
+public:
+ explicit TIssuesBuffer(ui32 capacity);
+
+ ui32 GetAllAddedIssuesCount() const;
+ void Push(const TIssues& issues);
+ std::vector<TIssueInfo> Dump();
+
+private:
+ void Clear();
+
+private:
+ ui32 Capacity;
+ ui32 AllAddedIssuesCount = 0;
+
+ std::vector<TIssueInfo> Issues;
+ ui32 HeadIndex = 0;
+ ui32 ItemsCount = 0;
+};
+
+}
+}
diff --git a/ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp b/ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp
index f58cf80e68f..64ee3679af1 100644
--- a/ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp
+++ b/ydb/library/yql/dq/actors/compute/ut/dq_compute_issues_buffer_ut.cpp
@@ -1,87 +1,87 @@
#include <ydb/library/yql/dq/actors/compute/dq_compute_issues_buffer.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NYql::NDq {
-
-Y_UNIT_TEST_SUITE(TIssuesBufferTest) {
- Y_UNIT_TEST(TestEmpty) {
- TIssuesBuffer buffer(5);
-
- UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 0);
-
- const auto dumps = buffer.Dump();
- UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 0);
- }
-
- Y_UNIT_TEST(TestSimplePush) {
- TIssuesBuffer buffer(5);
-
- auto issues = TIssues({ TIssue("issue1") });
- buffer.Push(issues);
-
- UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 1);
-
- const auto dumps = buffer.Dump();
- UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 1);
- UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue1");
- }
-
- Y_UNIT_TEST(TestPushWithOverflow) {
- TIssuesBuffer buffer(2);
-
- auto issues1 = TIssues({ TIssue("issue1") });
- auto issues2 = TIssues({ TIssue("issue2") });
- auto issues3 = TIssues({ TIssue("issue3") });
- buffer.Push(issues1);
- buffer.Push(issues2);
- buffer.Push(issues3);
-
- UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 3);
-
- const auto dumps = buffer.Dump();
- UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 2);
- UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue2");
- UNIT_ASSERT_STRINGS_EQUAL(dumps.at(1).Issues.begin()->Message, "issue3");
- }
-
- Y_UNIT_TEST(TestSmallBuffer) {
- TIssuesBuffer buffer(1);
-
- auto issues1 = TIssues({ TIssue("issue1") });
- auto issues2 = TIssues({ TIssue("issue2") });
- auto issues3 = TIssues({ TIssue("issue3") });
- buffer.Push(issues1);
- buffer.Push(issues2);
- buffer.Push(issues3);
-
- UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 3);
-
- const auto dumps = buffer.Dump();
- UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 1);
- UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue3");
- }
-
- Y_UNIT_TEST(TestUseAfterDump) {
- TIssuesBuffer buffer(2);
-
- auto issues1 = TIssues({ TIssue("issue1") });
- buffer.Push(issues1);
- buffer.Dump();
-
- auto issues2 = TIssues({ TIssue("issue2") });
- auto issues3 = TIssues({ TIssue("issue3") });
- auto issues4 = TIssues({ TIssue("issue4") });
- buffer.Push(issues2);
- buffer.Push(issues3);
- buffer.Push(issues4);
-
- UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 4);
- const auto dumps = buffer.Dump();
- UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 2);
- UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue3");
- UNIT_ASSERT_STRINGS_EQUAL(dumps.at(1).Issues.begin()->Message, "issue4");
- }
-}
-
-} // namespace NYql::NDq
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NYql::NDq {
+
+Y_UNIT_TEST_SUITE(TIssuesBufferTest) {
+ Y_UNIT_TEST(TestEmpty) {
+ TIssuesBuffer buffer(5);
+
+ UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 0);
+
+ const auto dumps = buffer.Dump();
+ UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 0);
+ }
+
+ Y_UNIT_TEST(TestSimplePush) {
+ TIssuesBuffer buffer(5);
+
+ auto issues = TIssues({ TIssue("issue1") });
+ buffer.Push(issues);
+
+ UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 1);
+
+ const auto dumps = buffer.Dump();
+ UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 1);
+ UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue1");
+ }
+
+ Y_UNIT_TEST(TestPushWithOverflow) {
+ TIssuesBuffer buffer(2);
+
+ auto issues1 = TIssues({ TIssue("issue1") });
+ auto issues2 = TIssues({ TIssue("issue2") });
+ auto issues3 = TIssues({ TIssue("issue3") });
+ buffer.Push(issues1);
+ buffer.Push(issues2);
+ buffer.Push(issues3);
+
+ UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 3);
+
+ const auto dumps = buffer.Dump();
+ UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 2);
+ UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue2");
+ UNIT_ASSERT_STRINGS_EQUAL(dumps.at(1).Issues.begin()->Message, "issue3");
+ }
+
+ Y_UNIT_TEST(TestSmallBuffer) {
+ TIssuesBuffer buffer(1);
+
+ auto issues1 = TIssues({ TIssue("issue1") });
+ auto issues2 = TIssues({ TIssue("issue2") });
+ auto issues3 = TIssues({ TIssue("issue3") });
+ buffer.Push(issues1);
+ buffer.Push(issues2);
+ buffer.Push(issues3);
+
+ UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 3);
+
+ const auto dumps = buffer.Dump();
+ UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 1);
+ UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue3");
+ }
+
+ Y_UNIT_TEST(TestUseAfterDump) {
+ TIssuesBuffer buffer(2);
+
+ auto issues1 = TIssues({ TIssue("issue1") });
+ buffer.Push(issues1);
+ buffer.Dump();
+
+ auto issues2 = TIssues({ TIssue("issue2") });
+ auto issues3 = TIssues({ TIssue("issue3") });
+ auto issues4 = TIssues({ TIssue("issue4") });
+ buffer.Push(issues2);
+ buffer.Push(issues3);
+ buffer.Push(issues4);
+
+ UNIT_ASSERT_VALUES_EQUAL(buffer.GetAllAddedIssuesCount(), 4);
+ const auto dumps = buffer.Dump();
+ UNIT_ASSERT_VALUES_EQUAL(dumps.size(), 2);
+ UNIT_ASSERT_STRINGS_EQUAL(dumps.at(0).Issues.begin()->Message, "issue3");
+ UNIT_ASSERT_STRINGS_EQUAL(dumps.at(1).Issues.begin()->Message, "issue4");
+ }
+}
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/actors/compute/ut/ya.make b/ydb/library/yql/dq/actors/compute/ut/ya.make
index 9006163f952..edf1744493c 100644
--- a/ydb/library/yql/dq/actors/compute/ut/ya.make
+++ b/ydb/library/yql/dq/actors/compute/ut/ya.make
@@ -1,17 +1,17 @@
UNITTEST_FOR(ydb/library/yql/dq/actors/compute)
-
-OWNER(
- d-mokhnatkin
- g:yq
- g:yql
-)
-
-SRCS(
- dq_compute_issues_buffer_ut.cpp
-)
-
-PEERDIR(
- library/cpp/testing/unittest
-)
-
-END()
+
+OWNER(
+ d-mokhnatkin
+ g:yq
+ g:yql
+)
+
+SRCS(
+ dq_compute_issues_buffer_ut.cpp
+)
+
+PEERDIR(
+ library/cpp/testing/unittest
+)
+
+END()
diff --git a/ydb/library/yql/dq/actors/compute/ya.make b/ydb/library/yql/dq/actors/compute/ya.make
index e17b7d97f43..c4b20bd502d 100644
--- a/ydb/library/yql/dq/actors/compute/ya.make
+++ b/ydb/library/yql/dq/actors/compute/ya.make
@@ -12,7 +12,7 @@ SRCS(
dq_compute_actor_checkpoints.cpp
dq_compute_actor_io_actors_factory.cpp
dq_compute_actor_stats.cpp
- dq_compute_issues_buffer.cpp
+ dq_compute_issues_buffer.cpp
retry_queue.cpp
)
@@ -32,10 +32,10 @@ PEERDIR(
ydb/library/yql/minikql/computation
)
-YQL_LAST_ABI_VERSION()
+YQL_LAST_ABI_VERSION()
-END()
+END()
-RECURSE_FOR_TESTS(
- ut
-)
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/dq/actors/protos/dq_stats.proto b/ydb/library/yql/dq/actors/protos/dq_stats.proto
index 6df0f4056cb..6b608790305 100644
--- a/ydb/library/yql/dq/actors/protos/dq_stats.proto
+++ b/ydb/library/yql/dq/actors/protos/dq_stats.proto
@@ -100,12 +100,12 @@ message TDqTableStats {
google.protobuf.Any Extra = 100;
}
-message TDqMkqlStat {
- string Name = 1;
- int64 Value = 2;
- bool Deriv = 3;
-}
-
+message TDqMkqlStat {
+ string Name = 1;
+ int64 Value = 2;
+ bool Deriv = 3;
+}
+
message TDqTaskStats {
// basic stats
uint64 TaskId = 1;
@@ -128,7 +128,7 @@ message TDqTaskStats {
uint64 PendingInputTimeUs = 107; // time waiting input data
uint64 PendingOutputTimeUs = 108; // time waiting output data
uint64 FinishTimeUs = 109; // time in finished state // ComputeCpuTimeUs + PendingInputTimeUs + PendingOutputTimeUs + FinishTimeUs == 100% (or == const in aggregated graphs for several stages/tasks)
- repeated TDqMkqlStat MkqlStats = 110; // stats from mkql
+ repeated TDqMkqlStat MkqlStats = 110; // stats from mkql
message THistBucket {
double Bound = 1;
diff --git a/ydb/library/yql/dq/runtime/dq_output.h b/ydb/library/yql/dq/runtime/dq_output.h
index e0014d49821..e793ff5231b 100644
--- a/ydb/library/yql/dq/runtime/dq_output.h
+++ b/ydb/library/yql/dq/runtime/dq_output.h
@@ -2,12 +2,12 @@
#include <ydb/library/yql/minikql/mkql_node.h>
-#include <util/datetime/base.h>
+#include <util/datetime/base.h>
#include <util/generic/ptr.h>
namespace NYql {
namespace NDqProto {
-
+
class TCheckpoint;
class TTaskInput;
} // namespace NDqProto
@@ -18,19 +18,19 @@ class TUnboxedValue;
namespace NDq {
-struct TDqOutputStats {
+struct TDqOutputStats {
// basic stats
- ui64 Chunks = 0;
- ui64 Bytes = 0;
- ui64 RowsIn = 0;
- ui64 RowsOut = 0;
+ ui64 Chunks = 0;
+ ui64 Bytes = 0;
+ ui64 RowsIn = 0;
+ ui64 RowsOut = 0;
TInstant FirstRowIn;
// profile stats
- ui64 MaxMemoryUsage = 0;
- ui64 MaxRowsInMemory = 0;
-};
-
+ ui64 MaxMemoryUsage = 0;
+ ui64 MaxRowsInMemory = 0;
+};
+
class IDqOutput : public TSimpleRefCount<IDqOutput> {
public:
using TPtr = TIntrusivePtr<IDqOutput>;
@@ -50,10 +50,10 @@ public:
[[nodiscard]]
virtual bool HasData() const = 0;
virtual bool IsFinished() const = 0;
-
+
virtual NKikimr::NMiniKQL::TType* GetOutputType() const = 0;
- virtual const TDqOutputStats* GetStats() const = 0;
+ virtual const TDqOutputStats* GetStats() const = 0;
};
} // namespace NDq
diff --git a/ydb/library/yql/dq/runtime/dq_output_channel.h b/ydb/library/yql/dq/runtime/dq_output_channel.h
index 6d6793ccfd1..f7a4887d2d6 100644
--- a/ydb/library/yql/dq/runtime/dq_output_channel.h
+++ b/ydb/library/yql/dq/runtime/dq_output_channel.h
@@ -14,7 +14,7 @@
namespace NYql::NDq {
-struct TDqOutputChannelStats : TDqOutputStats {
+struct TDqOutputChannelStats : TDqOutputStats {
ui64 ChannelId = 0;
// profile stats
diff --git a/ydb/library/yql/dq/runtime/dq_sink.cpp b/ydb/library/yql/dq/runtime/dq_sink.cpp
index aca1234ba0a..ff86523ef48 100644
--- a/ydb/library/yql/dq/runtime/dq_sink.cpp
+++ b/ydb/library/yql/dq/runtime/dq_sink.cpp
@@ -49,32 +49,32 @@ public:
void Push(NUdf::TUnboxedValue&& value) override {
if (!BasicStats.FirstRowIn) {
BasicStats.FirstRowIn = TInstant::Now();
- }
-
+ }
+
if (ValuesPushed++ % 1000 == 0) {
ReestimateRowBytes(value);
}
Y_VERIFY(EstimatedRowBytes > 0);
Values.emplace_back(std::move(value), EstimatedRowBytes);
EstimatedStoredBytes += EstimatedRowBytes;
-
- ReportChunkIn();
+
+ ReportChunkIn();
}
void Push(NDqProto::TCheckpoint&& checkpoint) override {
const ui64 bytesSize = checkpoint.ByteSize();
Values.emplace_back(std::move(checkpoint), bytesSize);
EstimatedStoredBytes += bytesSize;
-
- ReportChunkIn();
+
+ ReportChunkIn();
}
void Finish() override {
Finished = true;
-
+
if (!BasicStats.FirstRowIn) {
BasicStats.FirstRowIn = TInstant::Now();
- }
+ }
}
ui64 Pop(NKikimr::NMiniKQL::TUnboxedValueVector& batch, ui64 bytes) override {
@@ -99,22 +99,22 @@ public:
}
Y_VERIFY(EstimatedStoredBytes >= usedBytes);
EstimatedStoredBytes -= usedBytes;
-
- ReportChunkOut(batch.size(), usedBytes);
-
+
+ ReportChunkOut(batch.size(), usedBytes);
+
return usedBytes;
}
bool Pop(NDqProto::TCheckpoint& checkpoint) override {
if (!Values.empty() && std::holds_alternative<NDqProto::TCheckpoint>(Values.front().Value)) {
checkpoint = std::move(std::get<NDqProto::TCheckpoint>(Values.front().Value));
- const auto size = Values.front().EstimatedSize;
- Y_VERIFY(EstimatedStoredBytes >= size);
- EstimatedStoredBytes -= size;
+ const auto size = Values.front().EstimatedSize;
+ Y_VERIFY(EstimatedStoredBytes >= size);
+ EstimatedStoredBytes -= size;
Values.pop_front();
-
- ReportChunkOut(1, size);
-
+
+ ReportChunkOut(1, size);
+
return true;
}
return false;
@@ -141,10 +141,10 @@ public:
return OutputType;
}
- const TDqSinkStats* GetStats() const override {
+ const TDqSinkStats* GetStats() const override {
return &BasicStats;
- }
-
+ }
+
private:
void ReestimateRowBytes(const NUdf::TUnboxedValue& value) {
const ui64 valueSize = TDqDataSerializer::EstimateSize(value, OutputType);
@@ -158,20 +158,20 @@ private:
}
}
- void ReportChunkIn() {
+ void ReportChunkIn() {
BasicStats.Bytes += EstimatedRowBytes;
BasicStats.RowsIn++;
if (ProfileStats) {
ProfileStats->MaxMemoryUsage = std::max(ProfileStats->MaxMemoryUsage, EstimatedStoredBytes);
ProfileStats->MaxRowsInMemory = std::max(ProfileStats->MaxRowsInMemory, Values.size());
- }
- }
-
+ }
+ }
+
void ReportChunkOut(ui64 rowsCount, ui64 /* usedBytes */) {
BasicStats.Chunks++;
BasicStats.RowsOut += rowsCount;
- }
-
+ }
+
private:
const ui64 OutputIndex;
const ui64 MaxStoredBytes;
diff --git a/ydb/library/yql/dq/runtime/dq_sink.h b/ydb/library/yql/dq/runtime/dq_sink.h
index 6865ed37d2a..63c91c96602 100644
--- a/ydb/library/yql/dq/runtime/dq_sink.h
+++ b/ydb/library/yql/dq/runtime/dq_sink.h
@@ -7,13 +7,13 @@
namespace NYql::NDq {
-struct TDqSinkStats : TDqOutputStats {
+struct TDqSinkStats : TDqOutputStats {
const ui64 OutputIndex;
-
- explicit TDqSinkStats(ui64 outputIndex)
- : OutputIndex(outputIndex) {}
-};
-
+
+ explicit TDqSinkStats(ui64 outputIndex)
+ : OutputIndex(outputIndex) {}
+};
+
class IDqSink : public IDqOutput {
public:
using TPtr = TIntrusivePtr<IDqSink>;
@@ -26,8 +26,8 @@ public:
// Pop chechpoint. Checkpoints may be taken from sink even after it is finished.
[[nodiscard]]
virtual bool Pop(NDqProto::TCheckpoint& checkpoint) = 0;
-
- virtual const TDqSinkStats* GetStats() const = 0;
+
+ virtual const TDqSinkStats* GetStats() const = 0;
};
IDqSink::TPtr CreateDqSink(ui64 outputIndex, NKikimr::NMiniKQL::TType* outputType, ui64 maxStoredBytes,
diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
index 433c3b4fb6e..10900352c3b 100644
--- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
+++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
@@ -445,8 +445,8 @@ public:
TVector<IDqOutput::TPtr> outputs{Reserve(std::max<ui64>(outputDesc.ChannelsSize(), 1))};
if (outputDesc.HasSink()) {
- auto sink = CreateDqSink(i, ProgramParsed.OutputItemTypes[i], memoryLimits.ChannelBufferSize,
- Settings.CollectProfileStats);
+ auto sink = CreateDqSink(i, ProgramParsed.OutputItemTypes[i], memoryLimits.ChannelBufferSize,
+ Settings.CollectProfileStats);
auto [_, inserted] = Sinks.emplace(i, sink);
Y_VERIFY(inserted);
outputs.emplace_back(sink);
@@ -523,13 +523,13 @@ public:
if (Y_UNLIKELY(CollectProfileStats)) {
Stats->ComputeCpuTimeByRun->Collect(RunComputeTime.MilliSeconds());
-
- if (ProgramParsed.StatsRegistry) {
- Stats->MkqlStats.clear();
- ProgramParsed.StatsRegistry->ForEachStat([this](const TStatKey& key, i64 value) {
- Stats->MkqlStats.emplace_back(TMkqlStat{key, value});
- });
- }
+
+ if (ProgramParsed.StatsRegistry) {
+ Stats->MkqlStats.clear();
+ ProgramParsed.StatsRegistry->ForEachStat([this](const TStatKey& key, i64 value) {
+ Stats->MkqlStats.emplace_back(TMkqlStat{key, value});
+ });
+ }
}
if (runStatus == ERunStatus::Finished) {
diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.h b/ydb/library/yql/dq/runtime/dq_tasks_runner.h
index 46e4e4068e3..a0cc816e91e 100644
--- a/ydb/library/yql/dq/runtime/dq_tasks_runner.h
+++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.h
@@ -61,11 +61,11 @@ private:
TDuration StatusTime[StatusesCount];
};
-struct TMkqlStat {
- NKikimr::NMiniKQL::TStatKey Key;
- i64 Value = 0;
-};
-
+struct TMkqlStat {
+ NKikimr::NMiniKQL::TStatKey Key;
+ i64 Value = 0;
+};
+
struct TDqTaskRunnerStats {
// basic stats
TDuration BuildCpuTime;
@@ -83,8 +83,8 @@ struct TDqTaskRunnerStats {
THashMap<ui64, const TDqInputChannelStats*> InputChannels; // Channel id -> Channel stats
THashMap<ui64, const TDqSourceStats*> Sources; // Input index -> Source stats
THashMap<ui64, const TDqOutputChannelStats*> OutputChannels; // Channel id -> Channel stats
-
- TVector<TMkqlStat> MkqlStats;
+
+ TVector<TMkqlStat> MkqlStats;
};
struct TDqTaskRunnerContext {
diff --git a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
index 8c7c2ad2a41..fcbd3bb9ecf 100644
--- a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
+++ b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
@@ -151,10 +151,10 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
if (TDqStageBase::idx_Sinks < stage->ChildrenSize()) {
for (const auto& sink: stage->Child(TDqStageBase::idx_Sinks)->Children()) {
- sink->SetTypeAnn(resultType);
- }
- }
-
+ sink->SetTypeAnn(resultType);
+ }
+ }
+
stage->SetTypeAnn(ctx.MakeType<TTupleExprType>(resultTypesTuple));
return TStatus::Ok;
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
index d3ad89e1f19..57599acb641 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
@@ -51,7 +51,7 @@
#include "mkql_lookup.h"
#include "mkql_map.h"
#include "mkql_map_join.h"
-#include "mkql_multihopping.h"
+#include "mkql_multihopping.h"
#include "mkql_multimap.h"
#include "mkql_next_value.h"
#include "mkql_now.h"
@@ -220,7 +220,7 @@ struct TCallableComputationNodeBuilderFuncMapFiller {
{"CombineCore", &WrapCombineCore},
{"GroupingCore", &WrapGroupingCore},
{"HoppingCore", &WrapHoppingCore},
- {"MultiHoppingCore", &WrapMultiHoppingCore},
+ {"MultiHoppingCore", &WrapMultiHoppingCore},
{"ToBytes", &WrapToBytes},
{"FromBytes", &WrapFromBytes},
{"NewMTRand", &WrapNewMTRand},
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp
index bea9d861f3a..cb2b13f86d0 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp
@@ -1,431 +1,431 @@
-#include "mkql_multihopping.h"
-#include "mkql_saveload.h"
-
+#include "mkql_multihopping.h"
+#include "mkql_saveload.h"
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_stats_registry.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/minikql/watermark_tracker.h>
-
-#include <util/generic/scope.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
-
-const TStatKey Hop_NewHopsCount("MultiHop_NewHopsCount", true);
-const TStatKey Hop_ThrownEventsCount("MultiHop_ThrownEventsCount", true);
-const TStatKey Hop_EmptyTimeCount("MultiHop_EmptyTimeCount", true);
+
+#include <util/generic/scope.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+const TStatKey Hop_NewHopsCount("MultiHop_NewHopsCount", true);
+const TStatKey Hop_ThrownEventsCount("MultiHop_ThrownEventsCount", true);
+const TStatKey Hop_EmptyTimeCount("MultiHop_EmptyTimeCount", true);
const TStatKey Hop_KeysCount("MultiHop_KeysCount", true);
-
-constexpr ui32 StateVersion = 1;
-
-using TEqualsFunc = std::function<bool(NUdf::TUnboxedValuePod, NUdf::TUnboxedValuePod)>;
-using THashFunc = std::function<NYql::NUdf::THashType(NUdf::TUnboxedValuePod)>;
-
+
+constexpr ui32 StateVersion = 1;
+
+using TEqualsFunc = std::function<bool(NUdf::TUnboxedValuePod, NUdf::TUnboxedValuePod)>;
+using THashFunc = std::function<NYql::NUdf::THashType(NUdf::TUnboxedValuePod)>;
+
class TMultiHoppingCoreWrapper : public TStatefulSourceComputationNode<TMultiHoppingCoreWrapper, true> {
using TBaseComputation = TStatefulSourceComputationNode<TMultiHoppingCoreWrapper, true>;
-public:
- using TSelf = TMultiHoppingCoreWrapper;
-
- class TStreamValue : public TComputationValue<TStreamValue> {
- public:
- using TBase = TComputationValue<TStreamValue>;
-
- TStreamValue(
- TMemoryUsageInfo* memInfo,
- NUdf::TUnboxedValue&& stream,
- const TSelf* self,
- ui64 hopTime,
- ui64 intervalHopCount,
- ui64 delayHopCount,
- bool dataWatermarks,
- TComputationContext& ctx,
- const THashFunc& hash,
- const TEqualsFunc& equal)
- : TBase(memInfo)
- , Stream(std::move(stream))
- , Self(self)
- , HopTime(hopTime)
- , IntervalHopCount(intervalHopCount)
- , DelayHopCount(delayHopCount)
- , StatesMap(0, hash, equal)
- , Ctx(ctx)
- {
- if (dataWatermarks) {
- WatermarkTracker.emplace(TWatermarkTracker(delayHopCount * hopTime, hopTime));
- }
- }
-
- ~TStreamValue() {
- ClearState();
- }
-
- private:
- struct TBucket {
- NUdf::TUnboxedValue Value;
- bool HasValue = false;
- };
-
- struct TKeyState {
- std::vector<TBucket, TMKQLAllocator<TBucket>> Buckets; // circular buffer
- ui64 HopIndex; // Start index of current window
-
- TKeyState(ui64 bucketsCount, ui64 hopIndex)
- : Buckets(bucketsCount)
- , HopIndex(hopIndex)
- {}
-
- TKeyState(TKeyState&& state)
- : Buckets(std::move(state.Buckets))
- , HopIndex(state.HopIndex)
- {}
- };
-
- ui32 GetTraverseCount() const override {
- return 1;
- }
-
- NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
- Y_UNUSED(index);
- return Stream;
- }
-
- NUdf::TUnboxedValue Save() const override {
- MKQL_ENSURE(Ready.empty(), "Inconsistent state to save, not all elements are fetched");
-
- TString out;
- WriteUi32(out, StateVersion);
- WriteUi32(out, StatesMap.size());
- for (const auto& [key, state] : StatesMap) {
- WriteUnboxedValue(out, Self->KeyPacker.RefMutableObject(Ctx, false, Self->KeyType), key);
- WriteUi64(out, state.HopIndex);
- WriteUi32(out, state.Buckets.size());
- for (const auto& bucket : state.Buckets) {
- WriteBool(out, bucket.HasValue);
- if (bucket.HasValue) {
- Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
- if (Self->StateType) {
- WriteUnboxedValue(out, Self->StatePacker.RefMutableObject(Ctx, false, Self->StateType),
- Self->OutSave->GetValue(Ctx));
- }
- }
- }
- }
-
- WriteBool(out, Finished);
-
- auto strRef = NUdf::TStringRef(out.data(), out.size());
- return MakeString(strRef);
- }
-
- void Load(const NUdf::TStringRef& state) override {
- TStringBuf in(state.Data(), state.Size());
-
- const auto stateVersion = ReadUi32(in);
- if (stateVersion == 1) {
- const auto statesMapSize = ReadUi32(in);
- ClearState();
- StatesMap.reserve(statesMapSize);
- for (int i = 0; i < statesMapSize; i++) {
- auto key = ReadUnboxedValue(in, Self->KeyPacker.RefMutableObject(Ctx, false, Self->KeyType), Ctx);
- const auto hopIndex = ReadUi64(in);
- const auto bucketsSize = ReadUi32(in);
-
- TKeyState keyState(bucketsSize, hopIndex);
- for (auto& bucket : keyState.Buckets) {
- bucket.HasValue = ReadBool(in);
- if (bucket.HasValue) {
- if (Self->StateType) {
- Self->InLoad->SetValue(Ctx, ReadUnboxedValue(in, Self->StatePacker.RefMutableObject(Ctx, false, Self->StateType), Ctx));
- }
- bucket.Value = Self->OutLoad->GetValue(Ctx);
- }
- }
- StatesMap.emplace(key.Release(), std::move(keyState));
- }
-
- Finished = ReadBool(in);
- } else {
- THROW yexception() << "Invalid state version " << stateVersion;
- }
- }
-
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
- if (!Ready.empty()) {
- result = std::move(Ready.front());
- Ready.pop_front();
- return NUdf::EFetchStatus::Ok;
- }
-
- if (Finished) {
- return NUdf::EFetchStatus::Finish;
- }
-
- i64 thrownEventsStat = 0;
- i64 newHopsStat = 0;
- i64 emptyTimeCtStat = 0;
- Y_DEFER {
- MKQL_ADD_STAT(Ctx.Stats, Hop_ThrownEventsCount, thrownEventsStat);
- MKQL_ADD_STAT(Ctx.Stats, Hop_NewHopsCount, newHopsStat);
- MKQL_ADD_STAT(Ctx.Stats, Hop_EmptyTimeCount, emptyTimeCtStat);
- };
-
- for (NUdf::TUnboxedValue item;;) {
- if (!Ready.empty()) {
- result = std::move(Ready.front());
- Ready.pop_front();
- return NUdf::EFetchStatus::Ok;
- }
-
- const auto status = Stream.Fetch(item);
- if (status != NUdf::EFetchStatus::Ok) {
- if (status == NUdf::EFetchStatus::Finish) {
- CloseOldBuckets(Max<ui64>(), newHopsStat);
- Finished = true;
- if (!Ready.empty()) {
- result = std::move(Ready.front());
- Ready.pop_front();
- return NUdf::EFetchStatus::Ok;
- }
- }
- return status;
- }
-
- Self->Item->SetValue(Ctx, std::move(item));
- auto key = Self->KeyExtract->GetValue(Ctx);
- const auto& time = Self->OutTime->GetValue(Ctx);
- if (!time) {
- ++emptyTimeCtStat;
- continue;
- }
-
- const auto ts = time.Get<ui64>();
- const auto hopIndex = ts / HopTime;
- auto& keyState = GetOrCreateKeyState(key, hopIndex);
-
- if (hopIndex < keyState.HopIndex) {
- ++thrownEventsStat;
- continue;
- }
-
- // Overflow is not possible, because of hopIndex is a product of a division
- auto closeBeforeIndex = Max<i64>(hopIndex + 1 - DelayHopCount - IntervalHopCount, 0);
-
- CloseOldBucketsForKey(key, keyState, closeBeforeIndex, newHopsStat);
-
- auto& bucket = keyState.Buckets[hopIndex % keyState.Buckets.size()];
- if (!bucket.HasValue) {
- bucket.Value = Self->OutInit->GetValue(Ctx);
- bucket.HasValue = true;
- } else {
- Self->Key->SetValue(Ctx, NUdf::TUnboxedValue(key));
- Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
- bucket.Value = Self->OutUpdate->GetValue(Ctx);
- }
-
- if (WatermarkTracker) {
- const auto newWatermark = WatermarkTracker->HandleNextEventTime(ts);
- if (newWatermark) {
- CloseOldBuckets(*newWatermark, newHopsStat);
- }
- }
+public:
+ using TSelf = TMultiHoppingCoreWrapper;
+
+ class TStreamValue : public TComputationValue<TStreamValue> {
+ public:
+ using TBase = TComputationValue<TStreamValue>;
+
+ TStreamValue(
+ TMemoryUsageInfo* memInfo,
+ NUdf::TUnboxedValue&& stream,
+ const TSelf* self,
+ ui64 hopTime,
+ ui64 intervalHopCount,
+ ui64 delayHopCount,
+ bool dataWatermarks,
+ TComputationContext& ctx,
+ const THashFunc& hash,
+ const TEqualsFunc& equal)
+ : TBase(memInfo)
+ , Stream(std::move(stream))
+ , Self(self)
+ , HopTime(hopTime)
+ , IntervalHopCount(intervalHopCount)
+ , DelayHopCount(delayHopCount)
+ , StatesMap(0, hash, equal)
+ , Ctx(ctx)
+ {
+ if (dataWatermarks) {
+ WatermarkTracker.emplace(TWatermarkTracker(delayHopCount * hopTime, hopTime));
+ }
+ }
+
+ ~TStreamValue() {
+ ClearState();
+ }
+
+ private:
+ struct TBucket {
+ NUdf::TUnboxedValue Value;
+ bool HasValue = false;
+ };
+
+ struct TKeyState {
+ std::vector<TBucket, TMKQLAllocator<TBucket>> Buckets; // circular buffer
+ ui64 HopIndex; // Start index of current window
+
+ TKeyState(ui64 bucketsCount, ui64 hopIndex)
+ : Buckets(bucketsCount)
+ , HopIndex(hopIndex)
+ {}
+
+ TKeyState(TKeyState&& state)
+ : Buckets(std::move(state.Buckets))
+ , HopIndex(state.HopIndex)
+ {}
+ };
+
+ ui32 GetTraverseCount() const override {
+ return 1;
+ }
+
+ NUdf::TUnboxedValue GetTraverseItem(ui32 index) const override {
+ Y_UNUSED(index);
+ return Stream;
+ }
+
+ NUdf::TUnboxedValue Save() const override {
+ MKQL_ENSURE(Ready.empty(), "Inconsistent state to save, not all elements are fetched");
+
+ TString out;
+ WriteUi32(out, StateVersion);
+ WriteUi32(out, StatesMap.size());
+ for (const auto& [key, state] : StatesMap) {
+ WriteUnboxedValue(out, Self->KeyPacker.RefMutableObject(Ctx, false, Self->KeyType), key);
+ WriteUi64(out, state.HopIndex);
+ WriteUi32(out, state.Buckets.size());
+ for (const auto& bucket : state.Buckets) {
+ WriteBool(out, bucket.HasValue);
+ if (bucket.HasValue) {
+ Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
+ if (Self->StateType) {
+ WriteUnboxedValue(out, Self->StatePacker.RefMutableObject(Ctx, false, Self->StateType),
+ Self->OutSave->GetValue(Ctx));
+ }
+ }
+ }
+ }
+
+ WriteBool(out, Finished);
+
+ auto strRef = NUdf::TStringRef(out.data(), out.size());
+ return MakeString(strRef);
+ }
+
+ void Load(const NUdf::TStringRef& state) override {
+ TStringBuf in(state.Data(), state.Size());
+
+ const auto stateVersion = ReadUi32(in);
+ if (stateVersion == 1) {
+ const auto statesMapSize = ReadUi32(in);
+ ClearState();
+ StatesMap.reserve(statesMapSize);
+ for (int i = 0; i < statesMapSize; i++) {
+ auto key = ReadUnboxedValue(in, Self->KeyPacker.RefMutableObject(Ctx, false, Self->KeyType), Ctx);
+ const auto hopIndex = ReadUi64(in);
+ const auto bucketsSize = ReadUi32(in);
+
+ TKeyState keyState(bucketsSize, hopIndex);
+ for (auto& bucket : keyState.Buckets) {
+ bucket.HasValue = ReadBool(in);
+ if (bucket.HasValue) {
+ if (Self->StateType) {
+ Self->InLoad->SetValue(Ctx, ReadUnboxedValue(in, Self->StatePacker.RefMutableObject(Ctx, false, Self->StateType), Ctx));
+ }
+ bucket.Value = Self->OutLoad->GetValue(Ctx);
+ }
+ }
+ StatesMap.emplace(key.Release(), std::move(keyState));
+ }
+
+ Finished = ReadBool(in);
+ } else {
+ THROW yexception() << "Invalid state version " << stateVersion;
+ }
+ }
+
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+ if (!Ready.empty()) {
+ result = std::move(Ready.front());
+ Ready.pop_front();
+ return NUdf::EFetchStatus::Ok;
+ }
+
+ if (Finished) {
+ return NUdf::EFetchStatus::Finish;
+ }
+
+ i64 thrownEventsStat = 0;
+ i64 newHopsStat = 0;
+ i64 emptyTimeCtStat = 0;
+ Y_DEFER {
+ MKQL_ADD_STAT(Ctx.Stats, Hop_ThrownEventsCount, thrownEventsStat);
+ MKQL_ADD_STAT(Ctx.Stats, Hop_NewHopsCount, newHopsStat);
+ MKQL_ADD_STAT(Ctx.Stats, Hop_EmptyTimeCount, emptyTimeCtStat);
+ };
+
+ for (NUdf::TUnboxedValue item;;) {
+ if (!Ready.empty()) {
+ result = std::move(Ready.front());
+ Ready.pop_front();
+ return NUdf::EFetchStatus::Ok;
+ }
+
+ const auto status = Stream.Fetch(item);
+ if (status != NUdf::EFetchStatus::Ok) {
+ if (status == NUdf::EFetchStatus::Finish) {
+ CloseOldBuckets(Max<ui64>(), newHopsStat);
+ Finished = true;
+ if (!Ready.empty()) {
+ result = std::move(Ready.front());
+ Ready.pop_front();
+ return NUdf::EFetchStatus::Ok;
+ }
+ }
+ return status;
+ }
+
+ Self->Item->SetValue(Ctx, std::move(item));
+ auto key = Self->KeyExtract->GetValue(Ctx);
+ const auto& time = Self->OutTime->GetValue(Ctx);
+ if (!time) {
+ ++emptyTimeCtStat;
+ continue;
+ }
+
+ const auto ts = time.Get<ui64>();
+ const auto hopIndex = ts / HopTime;
+ auto& keyState = GetOrCreateKeyState(key, hopIndex);
+
+ if (hopIndex < keyState.HopIndex) {
+ ++thrownEventsStat;
+ continue;
+ }
+
+ // Overflow is not possible, because of hopIndex is a product of a division
+ auto closeBeforeIndex = Max<i64>(hopIndex + 1 - DelayHopCount - IntervalHopCount, 0);
+
+ CloseOldBucketsForKey(key, keyState, closeBeforeIndex, newHopsStat);
+
+ auto& bucket = keyState.Buckets[hopIndex % keyState.Buckets.size()];
+ if (!bucket.HasValue) {
+ bucket.Value = Self->OutInit->GetValue(Ctx);
+ bucket.HasValue = true;
+ } else {
+ Self->Key->SetValue(Ctx, NUdf::TUnboxedValue(key));
+ Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
+ bucket.Value = Self->OutUpdate->GetValue(Ctx);
+ }
+
+ if (WatermarkTracker) {
+ const auto newWatermark = WatermarkTracker->HandleNextEventTime(ts);
+ if (newWatermark) {
+ CloseOldBuckets(*newWatermark, newHopsStat);
+ }
+ }
MKQL_SET_STAT(Ctx.Stats, Hop_KeysCount, StatesMap.size());
- }
- }
-
- TKeyState& GetOrCreateKeyState(NUdf::TUnboxedValue& key, ui64 hopIndex) {
- const auto iter = StatesMap.try_emplace(
- key,
- IntervalHopCount + DelayHopCount,
- Max<i64>(hopIndex + 1 - IntervalHopCount, 0)
- // For first element we shouldn't forget windows in the past
- // Overflow is not possible, because of hopIndex is a product of a division
- );
- if (iter.second) {
- key.Ref();
- }
- return iter.first->second;
- }
-
- // Will return true if key state became empty
- bool CloseOldBucketsForKey(
- const NUdf::TUnboxedValue& key,
- TKeyState& keyState,
- const ui64 closeBeforeIndex, // Excluded bound
- i64& newHopsStat)
- {
- auto& bucketsForKey = keyState.Buckets;
-
- auto stateEmpty = true;
- for (auto i = 0; i < bucketsForKey.size(); i++) {
- const auto curHopIndex = keyState.HopIndex;
- if (curHopIndex >= closeBeforeIndex) {
- stateEmpty = false;
- break;
- }
-
- TMaybe<NUdf::TUnboxedValue> aggregated;
- for (ui64 j = 0; j < IntervalHopCount; j++) {
- const auto curBucketIndex = (curHopIndex + j) % bucketsForKey.size();
- const auto& bucket = bucketsForKey[curBucketIndex];
- if (bucket.HasValue) {
- if (!aggregated) { // todo: clone
- Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
- Self->InLoad->SetValue(Ctx, Self->OutSave->GetValue(Ctx));
- aggregated = Self->OutLoad->GetValue(Ctx);
- } else {
- Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
- Self->State2->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated));
- aggregated = Self->OutMerge->GetValue(Ctx);
- }
- }
- }
-
- auto& clearBucket = bucketsForKey[curHopIndex % bucketsForKey.size()];
- clearBucket.Value = NUdf::TUnboxedValue();
- clearBucket.HasValue = false;
-
- if (aggregated) {
- Self->Key->SetValue(Ctx, NUdf::TUnboxedValue(key));
- Self->State->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated));
- // Outer code requires window end time (not start as could be expected)
- Self->Time->SetValue(Ctx, NUdf::TUnboxedValuePod((curHopIndex + IntervalHopCount) * HopTime));
- Ready.emplace_back(Self->OutFinish->GetValue(Ctx));
- }
-
- keyState.HopIndex++;
- newHopsStat++;
- }
-
- return stateEmpty;
- }
-
- void CloseOldBuckets(ui64 watermarkTs, i64& newHops) {
- const auto watermarkIndex = watermarkTs / HopTime;
- EraseNodesIf(StatesMap, [&](auto& iter) {
- auto& [key, val] = iter;
- const auto keyStateBecameEmpty = CloseOldBucketsForKey(key, val, watermarkIndex + 1 - IntervalHopCount, newHops);
- if (keyStateBecameEmpty) {
- key.UnRef();
- }
- return keyStateBecameEmpty;
- });
- return;
- }
-
- void ClearState() {
- EraseNodesIf(StatesMap, [&](auto& iter) {
- iter.first.UnRef();
- return true;
- });
- StatesMap.rehash(0);
- }
-
- const NUdf::TUnboxedValue Stream;
- const TSelf *const Self;
-
- const ui64 HopTime;
- const ui64 IntervalHopCount;
- const ui64 DelayHopCount;
-
- using TStatesMap = std::unordered_map<
- NUdf::TUnboxedValuePod, TKeyState,
- THashFunc, TEqualsFunc,
- TMKQLAllocator<std::pair<const NUdf::TUnboxedValuePod, TKeyState>>>;
-
- TStatesMap StatesMap; // Map of states for each key
- std::deque<NUdf::TUnboxedValue> Ready; // buffer for fetching results
- bool Finished = false;
-
- TComputationContext& Ctx;
- std::optional<TWatermarkTracker> WatermarkTracker;
- };
-
- TMultiHoppingCoreWrapper(
- TComputationMutables& mutables,
- IComputationNode* stream,
- IComputationExternalNode* item,
- IComputationExternalNode* key,
- IComputationExternalNode* state,
- IComputationExternalNode* state2,
- IComputationExternalNode* time,
- IComputationExternalNode* inSave,
- IComputationExternalNode* inLoad,
- IComputationNode* keyExtract,
- IComputationNode* outTime,
- IComputationNode* outInit,
- IComputationNode* outUpdate,
- IComputationNode* outSave,
- IComputationNode* outLoad,
- IComputationNode* outMerge,
- IComputationNode* outFinish,
- IComputationNode* hop,
- IComputationNode* interval,
- IComputationNode* delay,
- IComputationNode* dataWatermarks,
- TType* keyType,
- TType* stateType)
- : TBaseComputation(mutables)
- , Stream(stream)
- , Item(item)
- , Key(key)
- , State(state)
- , State2(state2)
- , Time(time)
- , InSave(inSave)
- , InLoad(inLoad)
- , KeyExtract(keyExtract)
- , OutTime(outTime)
- , OutInit(outInit)
- , OutUpdate(outUpdate)
- , OutSave(outSave)
- , OutLoad(outLoad)
- , OutMerge(outMerge)
- , OutFinish(outFinish)
- , Hop(hop)
- , Interval(interval)
- , Delay(delay)
- , DataWatermarks(dataWatermarks)
- , KeyType(keyType)
- , StateType(stateType)
- , KeyPacker(mutables)
- , StatePacker(mutables)
- , KeyTypes()
- , IsTuple()
- {
- Stateless = false;
-
- bool encoded;
- GetDictionaryKeyTypes(keyType, KeyTypes, IsTuple, encoded);
- Y_VERIFY(!encoded, "TODO");
- }
-
+ }
+ }
+
+ TKeyState& GetOrCreateKeyState(NUdf::TUnboxedValue& key, ui64 hopIndex) {
+ const auto iter = StatesMap.try_emplace(
+ key,
+ IntervalHopCount + DelayHopCount,
+ Max<i64>(hopIndex + 1 - IntervalHopCount, 0)
+ // For first element we shouldn't forget windows in the past
+ // Overflow is not possible, because of hopIndex is a product of a division
+ );
+ if (iter.second) {
+ key.Ref();
+ }
+ return iter.first->second;
+ }
+
+ // Will return true if key state became empty
+ bool CloseOldBucketsForKey(
+ const NUdf::TUnboxedValue& key,
+ TKeyState& keyState,
+ const ui64 closeBeforeIndex, // Excluded bound
+ i64& newHopsStat)
+ {
+ auto& bucketsForKey = keyState.Buckets;
+
+ auto stateEmpty = true;
+ for (auto i = 0; i < bucketsForKey.size(); i++) {
+ const auto curHopIndex = keyState.HopIndex;
+ if (curHopIndex >= closeBeforeIndex) {
+ stateEmpty = false;
+ break;
+ }
+
+ TMaybe<NUdf::TUnboxedValue> aggregated;
+ for (ui64 j = 0; j < IntervalHopCount; j++) {
+ const auto curBucketIndex = (curHopIndex + j) % bucketsForKey.size();
+ const auto& bucket = bucketsForKey[curBucketIndex];
+ if (bucket.HasValue) {
+ if (!aggregated) { // todo: clone
+ Self->InSave->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
+ Self->InLoad->SetValue(Ctx, Self->OutSave->GetValue(Ctx));
+ aggregated = Self->OutLoad->GetValue(Ctx);
+ } else {
+ Self->State->SetValue(Ctx, NUdf::TUnboxedValue(bucket.Value));
+ Self->State2->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated));
+ aggregated = Self->OutMerge->GetValue(Ctx);
+ }
+ }
+ }
+
+ auto& clearBucket = bucketsForKey[curHopIndex % bucketsForKey.size()];
+ clearBucket.Value = NUdf::TUnboxedValue();
+ clearBucket.HasValue = false;
+
+ if (aggregated) {
+ Self->Key->SetValue(Ctx, NUdf::TUnboxedValue(key));
+ Self->State->SetValue(Ctx, NUdf::TUnboxedValue(*aggregated));
+ // Outer code requires window end time (not start as could be expected)
+ Self->Time->SetValue(Ctx, NUdf::TUnboxedValuePod((curHopIndex + IntervalHopCount) * HopTime));
+ Ready.emplace_back(Self->OutFinish->GetValue(Ctx));
+ }
+
+ keyState.HopIndex++;
+ newHopsStat++;
+ }
+
+ return stateEmpty;
+ }
+
+ void CloseOldBuckets(ui64 watermarkTs, i64& newHops) {
+ const auto watermarkIndex = watermarkTs / HopTime;
+ EraseNodesIf(StatesMap, [&](auto& iter) {
+ auto& [key, val] = iter;
+ const auto keyStateBecameEmpty = CloseOldBucketsForKey(key, val, watermarkIndex + 1 - IntervalHopCount, newHops);
+ if (keyStateBecameEmpty) {
+ key.UnRef();
+ }
+ return keyStateBecameEmpty;
+ });
+ return;
+ }
+
+ void ClearState() {
+ EraseNodesIf(StatesMap, [&](auto& iter) {
+ iter.first.UnRef();
+ return true;
+ });
+ StatesMap.rehash(0);
+ }
+
+ const NUdf::TUnboxedValue Stream;
+ const TSelf *const Self;
+
+ const ui64 HopTime;
+ const ui64 IntervalHopCount;
+ const ui64 DelayHopCount;
+
+ using TStatesMap = std::unordered_map<
+ NUdf::TUnboxedValuePod, TKeyState,
+ THashFunc, TEqualsFunc,
+ TMKQLAllocator<std::pair<const NUdf::TUnboxedValuePod, TKeyState>>>;
+
+ TStatesMap StatesMap; // Map of states for each key
+ std::deque<NUdf::TUnboxedValue> Ready; // buffer for fetching results
+ bool Finished = false;
+
+ TComputationContext& Ctx;
+ std::optional<TWatermarkTracker> WatermarkTracker;
+ };
+
+ TMultiHoppingCoreWrapper(
+ TComputationMutables& mutables,
+ IComputationNode* stream,
+ IComputationExternalNode* item,
+ IComputationExternalNode* key,
+ IComputationExternalNode* state,
+ IComputationExternalNode* state2,
+ IComputationExternalNode* time,
+ IComputationExternalNode* inSave,
+ IComputationExternalNode* inLoad,
+ IComputationNode* keyExtract,
+ IComputationNode* outTime,
+ IComputationNode* outInit,
+ IComputationNode* outUpdate,
+ IComputationNode* outSave,
+ IComputationNode* outLoad,
+ IComputationNode* outMerge,
+ IComputationNode* outFinish,
+ IComputationNode* hop,
+ IComputationNode* interval,
+ IComputationNode* delay,
+ IComputationNode* dataWatermarks,
+ TType* keyType,
+ TType* stateType)
+ : TBaseComputation(mutables)
+ , Stream(stream)
+ , Item(item)
+ , Key(key)
+ , State(state)
+ , State2(state2)
+ , Time(time)
+ , InSave(inSave)
+ , InLoad(inLoad)
+ , KeyExtract(keyExtract)
+ , OutTime(outTime)
+ , OutInit(outInit)
+ , OutUpdate(outUpdate)
+ , OutSave(outSave)
+ , OutLoad(outLoad)
+ , OutMerge(outMerge)
+ , OutFinish(outFinish)
+ , Hop(hop)
+ , Interval(interval)
+ , Delay(delay)
+ , DataWatermarks(dataWatermarks)
+ , KeyType(keyType)
+ , StateType(stateType)
+ , KeyPacker(mutables)
+ , StatePacker(mutables)
+ , KeyTypes()
+ , IsTuple()
+ {
+ Stateless = false;
+
+ bool encoded;
+ GetDictionaryKeyTypes(keyType, KeyTypes, IsTuple, encoded);
+ Y_VERIFY(!encoded, "TODO");
+ }
+
NUdf::TUnboxedValuePod CreateStream(TComputationContext& ctx) const {
- const auto hopTime = Hop->GetValue(ctx).Get<i64>();
- const auto interval = Interval->GetValue(ctx).Get<i64>();
- const auto delay = Delay->GetValue(ctx).Get<i64>();
- const auto dataWatermarks = DataWatermarks->GetValue(ctx).Get<bool>();
-
- // TODO: move checks from here
- MKQL_ENSURE(hopTime > 0, "hop must be positive");
- MKQL_ENSURE(interval >= hopTime, "interval should be greater or equal to hop");
- MKQL_ENSURE(delay >= hopTime, "delay should be greater or equal to hop");
-
- const auto intervalHopCount = interval / hopTime;
- const auto delayHopCount = delay / hopTime;
-
- MKQL_ENSURE(intervalHopCount <= 100000, "too many hops in interval");
- MKQL_ENSURE(delayHopCount <= 100000, "too many hops in delay");
-
- return ctx.HolderFactory.Create<TStreamValue>(Stream->GetValue(ctx), this, (ui64)hopTime,
- (ui64)intervalHopCount, (ui64)delayHopCount,
- dataWatermarks, ctx,
- TValueHasher(KeyTypes, IsTuple),
- TValueEqual(KeyTypes, IsTuple));
- }
-
+ const auto hopTime = Hop->GetValue(ctx).Get<i64>();
+ const auto interval = Interval->GetValue(ctx).Get<i64>();
+ const auto delay = Delay->GetValue(ctx).Get<i64>();
+ const auto dataWatermarks = DataWatermarks->GetValue(ctx).Get<bool>();
+
+ // TODO: move checks from here
+ MKQL_ENSURE(hopTime > 0, "hop must be positive");
+ MKQL_ENSURE(interval >= hopTime, "interval should be greater or equal to hop");
+ MKQL_ENSURE(delay >= hopTime, "delay should be greater or equal to hop");
+
+ const auto intervalHopCount = interval / hopTime;
+ const auto delayHopCount = delay / hopTime;
+
+ MKQL_ENSURE(intervalHopCount <= 100000, "too many hops in interval");
+ MKQL_ENSURE(delayHopCount <= 100000, "too many hops in delay");
+
+ return ctx.HolderFactory.Create<TStreamValue>(Stream->GetValue(ctx), this, (ui64)hopTime,
+ (ui64)intervalHopCount, (ui64)delayHopCount,
+ dataWatermarks, ctx,
+ TValueHasher(KeyTypes, IsTuple),
+ TValueEqual(KeyTypes, IsTuple));
+ }
+
NUdf::TUnboxedValue GetValue(TComputationContext& compCtx) const override {
NUdf::TUnboxedValue& valueRef = ValueRef(compCtx);
if (valueRef.IsInvalid()) {
@@ -441,115 +441,115 @@ public:
return valueRef;
}
-private:
- void RegisterDependencies() const final {
- DependsOn(Stream);
- Own(Item);
- Own(Key);
- Own(State);
- Own(State2);
- Own(Time);
- Own(InSave);
- Own(InLoad);
- DependsOn(KeyExtract);
- DependsOn(OutTime);
- DependsOn(OutInit);
- DependsOn(OutUpdate);
- DependsOn(OutSave);
- DependsOn(OutLoad);
- DependsOn(OutMerge);
- DependsOn(OutFinish);
- DependsOn(Hop);
- DependsOn(Interval);
- DependsOn(Delay);
- DependsOn(DataWatermarks);
- }
-
- IComputationNode* const Stream;
-
- IComputationExternalNode* const Item;
- IComputationExternalNode* const Key;
- IComputationExternalNode* const State;
- IComputationExternalNode* const State2;
- IComputationExternalNode* const Time;
- IComputationExternalNode* const InSave;
- IComputationExternalNode* const InLoad;
-
- IComputationNode* const KeyExtract;
- IComputationNode* const OutTime;
- IComputationNode* const OutInit;
- IComputationNode* const OutUpdate;
- IComputationNode* const OutSave;
- IComputationNode* const OutLoad;
- IComputationNode* const OutMerge;
- IComputationNode* const OutFinish;
-
- IComputationNode* const Hop;
- IComputationNode* const Interval;
- IComputationNode* const Delay;
- IComputationNode* const DataWatermarks;
-
- TType* const KeyType;
- TType* const StateType;
- TMutableObjectOverBoxedValue<TValuePackerBoxed> KeyPacker;
- TMutableObjectOverBoxedValue<TValuePackerBoxed> StatePacker;
-
- TKeyTypes KeyTypes;
- bool IsTuple;
-};
-
-}
-
-IComputationNode* WrapMultiHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 20, "Expected 20 args");
-
- auto hasSaveLoad = !callable.GetInput(12).GetStaticType()->IsVoid();
-
- IComputationExternalNode* inSave = nullptr;
- IComputationNode* outSave = nullptr;
- IComputationExternalNode* inLoad = nullptr;
- IComputationNode* outLoad = nullptr;
-
- auto streamType = callable.GetInput(0).GetStaticType();
- MKQL_ENSURE(streamType->IsStream(), "Expected stream");
-
- const auto keyType = callable.GetInput(8).GetStaticType();
-
- auto stream = LocateNode(ctx.NodeLocator, callable, 0);
-
- auto keyExtract = LocateNode(ctx.NodeLocator, callable, 8);
- auto outTime = LocateNode(ctx.NodeLocator, callable, 9);
- auto outInit = LocateNode(ctx.NodeLocator, callable, 10);
- auto outUpdate = LocateNode(ctx.NodeLocator, callable, 11);
- if (hasSaveLoad) {
- outSave = LocateNode(ctx.NodeLocator, callable, 12);
- outLoad = LocateNode(ctx.NodeLocator, callable, 13);
- }
- auto outMerge = LocateNode(ctx.NodeLocator, callable, 14);
- auto outFinish = LocateNode(ctx.NodeLocator, callable, 15);
-
- auto hop = LocateNode(ctx.NodeLocator, callable, 16);
- auto interval = LocateNode(ctx.NodeLocator, callable, 17);
- auto delay = LocateNode(ctx.NodeLocator, callable, 18);
- auto dataWatermarks = LocateNode(ctx.NodeLocator, callable, 19);
-
- auto item = LocateExternalNode(ctx.NodeLocator, callable, 1);
- auto key = LocateExternalNode(ctx.NodeLocator, callable, 2);
- auto state = LocateExternalNode(ctx.NodeLocator, callable, 3);
- auto state2 = LocateExternalNode(ctx.NodeLocator, callable, 4);
- auto time = LocateExternalNode(ctx.NodeLocator, callable, 5);
- if (hasSaveLoad) {
- inSave = LocateExternalNode(ctx.NodeLocator, callable, 6);
- inLoad = LocateExternalNode(ctx.NodeLocator, callable, 7);
- }
-
- auto stateType = hasSaveLoad ? callable.GetInput(12).GetStaticType() : nullptr;
-
- return new TMultiHoppingCoreWrapper(ctx.Mutables,
- stream, item, key, state, state2, time, inSave, inLoad, keyExtract,
- outTime, outInit, outUpdate, outSave, outLoad, outMerge, outFinish,
- hop, interval, delay, dataWatermarks, keyType, stateType);
-}
-
-}
-}
+private:
+ void RegisterDependencies() const final {
+ DependsOn(Stream);
+ Own(Item);
+ Own(Key);
+ Own(State);
+ Own(State2);
+ Own(Time);
+ Own(InSave);
+ Own(InLoad);
+ DependsOn(KeyExtract);
+ DependsOn(OutTime);
+ DependsOn(OutInit);
+ DependsOn(OutUpdate);
+ DependsOn(OutSave);
+ DependsOn(OutLoad);
+ DependsOn(OutMerge);
+ DependsOn(OutFinish);
+ DependsOn(Hop);
+ DependsOn(Interval);
+ DependsOn(Delay);
+ DependsOn(DataWatermarks);
+ }
+
+ IComputationNode* const Stream;
+
+ IComputationExternalNode* const Item;
+ IComputationExternalNode* const Key;
+ IComputationExternalNode* const State;
+ IComputationExternalNode* const State2;
+ IComputationExternalNode* const Time;
+ IComputationExternalNode* const InSave;
+ IComputationExternalNode* const InLoad;
+
+ IComputationNode* const KeyExtract;
+ IComputationNode* const OutTime;
+ IComputationNode* const OutInit;
+ IComputationNode* const OutUpdate;
+ IComputationNode* const OutSave;
+ IComputationNode* const OutLoad;
+ IComputationNode* const OutMerge;
+ IComputationNode* const OutFinish;
+
+ IComputationNode* const Hop;
+ IComputationNode* const Interval;
+ IComputationNode* const Delay;
+ IComputationNode* const DataWatermarks;
+
+ TType* const KeyType;
+ TType* const StateType;
+ TMutableObjectOverBoxedValue<TValuePackerBoxed> KeyPacker;
+ TMutableObjectOverBoxedValue<TValuePackerBoxed> StatePacker;
+
+ TKeyTypes KeyTypes;
+ bool IsTuple;
+};
+
+}
+
+IComputationNode* WrapMultiHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ MKQL_ENSURE(callable.GetInputsCount() == 20, "Expected 20 args");
+
+ auto hasSaveLoad = !callable.GetInput(12).GetStaticType()->IsVoid();
+
+ IComputationExternalNode* inSave = nullptr;
+ IComputationNode* outSave = nullptr;
+ IComputationExternalNode* inLoad = nullptr;
+ IComputationNode* outLoad = nullptr;
+
+ auto streamType = callable.GetInput(0).GetStaticType();
+ MKQL_ENSURE(streamType->IsStream(), "Expected stream");
+
+ const auto keyType = callable.GetInput(8).GetStaticType();
+
+ auto stream = LocateNode(ctx.NodeLocator, callable, 0);
+
+ auto keyExtract = LocateNode(ctx.NodeLocator, callable, 8);
+ auto outTime = LocateNode(ctx.NodeLocator, callable, 9);
+ auto outInit = LocateNode(ctx.NodeLocator, callable, 10);
+ auto outUpdate = LocateNode(ctx.NodeLocator, callable, 11);
+ if (hasSaveLoad) {
+ outSave = LocateNode(ctx.NodeLocator, callable, 12);
+ outLoad = LocateNode(ctx.NodeLocator, callable, 13);
+ }
+ auto outMerge = LocateNode(ctx.NodeLocator, callable, 14);
+ auto outFinish = LocateNode(ctx.NodeLocator, callable, 15);
+
+ auto hop = LocateNode(ctx.NodeLocator, callable, 16);
+ auto interval = LocateNode(ctx.NodeLocator, callable, 17);
+ auto delay = LocateNode(ctx.NodeLocator, callable, 18);
+ auto dataWatermarks = LocateNode(ctx.NodeLocator, callable, 19);
+
+ auto item = LocateExternalNode(ctx.NodeLocator, callable, 1);
+ auto key = LocateExternalNode(ctx.NodeLocator, callable, 2);
+ auto state = LocateExternalNode(ctx.NodeLocator, callable, 3);
+ auto state2 = LocateExternalNode(ctx.NodeLocator, callable, 4);
+ auto time = LocateExternalNode(ctx.NodeLocator, callable, 5);
+ if (hasSaveLoad) {
+ inSave = LocateExternalNode(ctx.NodeLocator, callable, 6);
+ inLoad = LocateExternalNode(ctx.NodeLocator, callable, 7);
+ }
+
+ auto stateType = hasSaveLoad ? callable.GetInput(12).GetStaticType() : nullptr;
+
+ return new TMultiHoppingCoreWrapper(ctx.Mutables,
+ stream, item, key, state, state2, time, inSave, inLoad, keyExtract,
+ outTime, outInit, outUpdate, outSave, outLoad, outMerge, outFinish,
+ hop, interval, delay, dataWatermarks, keyType, stateType);
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h
index f0c7dbb6971..457b2ebe55a 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.h
@@ -1,11 +1,11 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-IComputationNode* WrapMultiHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-
-}
-}
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+IComputationNode* WrapMultiHoppingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx);
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
index 2947e9516a3..6819dc77187 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
@@ -1,300 +1,300 @@
-#include <ydb/library/yql/minikql/mkql_node.h>
-#include <ydb/library/yql/minikql/mkql_node_cast.h>
-#include <ydb/library/yql/minikql/mkql_program_builder.h>
-#include <ydb/library/yql/minikql/mkql_function_registry.h>
-#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
-#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
-#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h>
-#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
-#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
- TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
- return CreateDeterministicRandomProvider(1);
- }
-
- TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
- return CreateDeterministicTimeProvider(10000000);
- }
-
- TComputationNodeFactory GetAuxCallableFactory() {
- return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
- if (callable.GetType()->GetName() == "OneYieldStream") {
- return new TExternalComputationNode(ctx.Mutables);
- }
-
- return GetBuiltinFactory()(callable, ctx);
- };
- }
-
- struct TSetup {
- TSetup(TScopedAlloc& alloc)
- : Alloc(alloc)
- {
- FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
- RandomProvider = CreateRandomProvider();
- TimeProvider = CreateTimeProvider();
-
- Env.Reset(new TTypeEnvironment(Alloc));
- PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
- }
-
- THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
- Explorer.Walk(pgm.GetNode(), *Env);
- TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(),
- FunctionRegistry.Get(),
- NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi);
- Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
- TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
- return Pattern->Clone(compOpts);
- }
-
- TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
- TIntrusivePtr<IRandomProvider> RandomProvider;
- TIntrusivePtr<ITimeProvider> TimeProvider;
-
- TScopedAlloc& Alloc;
- THolder<TTypeEnvironment> Env;
- THolder<TProgramBuilder> PgmBuilder;
-
- TExploringNodeVisitor Explorer;
- IComputationPattern::TPtr Pattern;
- };
-
- struct TStreamWithYield : public NUdf::TBoxedValue {
- TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index)
- : Items(items)
- , YieldPos(yieldPos)
- , Index(index)
- {}
-
- private:
- TUnboxedValueVector Items;
- ui32 YieldPos;
- ui32 Index;
-
- ui32 GetTraverseCount() const override {
- return 0;
- }
-
- NUdf::TUnboxedValue Save() const override {
- return NUdf::TUnboxedValue::Zero();
- }
-
- void Load(const NUdf::TStringRef& state) override {
- Y_UNUSED(state);
- }
-
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
- if (Index >= Items.size()) {
- return NUdf::EFetchStatus::Finish;
- }
- if (Index == YieldPos) {
- return NUdf::EFetchStatus::Yield;
- }
- result = Items[Index++];
- return NUdf::EFetchStatus::Ok;
- }
- };
-
- THolder<IComputationGraph> BuildGraph(TSetup& setup, const std::vector<std::tuple<ui32, i64, ui32>> items,
- ui32 yieldPos, ui32 startIndex, bool dataWatermarks) {
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto structType = pgmBuilder.NewEmptyStructType();
- structType = pgmBuilder.NewStructType(structType, "key",
- pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
- structType = pgmBuilder.NewStructType(structType, "time",
- pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
- structType = pgmBuilder.NewStructType(structType, "sum",
- pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
- auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key");
- auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
- auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
-
- auto inStreamType = pgmBuilder.NewStreamType(structType);
-
- TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType);
- auto streamNode = inStream.Build();
-
- ui64 hop = 10, interval = 30, delay = 20;
-
- auto pgmReturn = pgmBuilder.MultiHoppingCore(
- TRuntimeNode(streamNode, false),
- [&](TRuntimeNode item) { // keyExtractor
- return pgmBuilder.Member(item, "key");
- },
- [&](TRuntimeNode item) { // timeExtractor
- return pgmBuilder.Member(item, "time");
- },
- [&](TRuntimeNode item) { // init
- std::vector<std::pair<std::string_view, TRuntimeNode>> members;
- members.emplace_back("sum", pgmBuilder.Member(item, "sum"));
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode item, TRuntimeNode state) { // update
- auto add = pgmBuilder.AggrAdd(
- pgmBuilder.Member(item, "sum"),
- pgmBuilder.Member(state, "sum"));
- std::vector<std::pair<std::string_view, TRuntimeNode>> members;
- members.emplace_back("sum", add);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode state) { // save
- return pgmBuilder.Member(state, "sum");
- },
- [&](TRuntimeNode savedState) { // load
- std::vector<std::pair<std::string_view, TRuntimeNode>> members;
- members.emplace_back("sum", savedState);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
- auto add = pgmBuilder.AggrAdd(
- pgmBuilder.Member(state1, "sum"),
- pgmBuilder.Member(state2, "sum"));
- std::vector<std::pair<std::string_view, TRuntimeNode>> members;
- members.emplace_back("sum", add);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish
- Y_UNUSED(time);
- std::vector<std::pair<std::string_view, TRuntimeNode>> members;
- members.emplace_back("key", key);
- members.emplace_back("sum", pgmBuilder.Member(state, "sum"));
- return pgmBuilder.NewStruct(members);
- },
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay
- pgmBuilder.NewDataLiteral<bool>(dataWatermarks) // dataWatermarks
- );
-
- auto graph = setup.BuildGraph(pgmReturn, {streamNode});
-
- TUnboxedValueVector streamItems;
- for (size_t i = 0; i < items.size(); ++i) {
- NUdf::TUnboxedValue* itemsPtr;
- auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr);
- itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(std::get<0>(items[i]));
- itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(std::get<1>(items[i]));
- itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(std::get<2>(items[i]));
- streamItems.push_back(std::move(structValues));
- }
-
- auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
- graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
- return graph;
- }
-}
-
-Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingSaveLoadTest) {
- void TestWithSaveLoadImpl(
- const std::vector<std::tuple<ui32, i64, ui32>> input,
- const std::vector<std::tuple<ui32, ui32>> expected,
- bool withTraverse,
- bool dataWatermarks)
- {
- TScopedAlloc alloc;
-
- for (ui32 yieldPos = 0; yieldPos < input.size(); ++yieldPos) {
- std::vector<std::tuple<ui32, ui32>> result;
-
- TSetup setup1(alloc);
- auto graph1 = BuildGraph(setup1, input, yieldPos, 0, dataWatermarks);
- auto root1 = graph1->GetValue();
-
- NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
- while (status == NUdf::EFetchStatus::Ok) {
- NUdf::TUnboxedValue val;
- status = root1.Fetch(val);
- if (status == NUdf::EFetchStatus::Ok) {
- result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>());
- }
- }
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
-
- TString graphState;
- if (withTraverse) {
- SaveGraphState(&root1, 1, 0ULL, graphState);
- } else {
- graphState = graph1->SaveGraphState();
- }
-
- TSetup setup2(alloc);
- auto graph2 = BuildGraph(setup2, input, -1, yieldPos, dataWatermarks);
- NUdf::TUnboxedValue root2;
- if (withTraverse) {
- root2 = graph2->GetValue();
- LoadGraphState(&root2, 1, 0ULL, graphState);
- } else {
- graph2->LoadGraphState(graphState);
- root2 = graph2->GetValue();
- }
-
- status = NUdf::EFetchStatus::Ok;
- while (status == NUdf::EFetchStatus::Ok) {
- NUdf::TUnboxedValue val;
- status = root2.Fetch(val);
- if (status == NUdf::EFetchStatus::Ok) {
- result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>());
- }
- }
- UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
-
- auto sortedExpected = expected;
- std::sort(result.begin(), result.end());
- std::sort(sortedExpected.begin(), sortedExpected.end());
- UNIT_ASSERT_EQUAL(result, sortedExpected);
- }
- }
-
- const std::vector<std::tuple<ui32, i64, ui32>> input1 = {
- // Group; Time; Value
- {2, 1, 2},
- {1, 1, 2},
- {2, 2, 3},
- {1, 2, 3},
- {2, 15, 4},
- {1, 15, 4},
- {2, 23, 6},
- {1, 23, 6},
- {2, 24, 5},
- {1, 24, 5},
- {2, 25, 7},
- {1, 25, 7},
- {2, 40, 2},
- {1, 40, 2},
- {2, 47, 1},
- {1, 47, 1},
- {2, 51, 6},
- {1, 51, 6},
- {2, 59, 2},
- {1, 59, 2},
- {2, 85, 8},
- {1, 85, 8}
- };
-
- const std::vector<std::tuple<ui32, ui32>> expected = {
- {1, 8}, {1, 8}, {1, 8}, {1, 8},
- {1, 11}, {1, 11}, {1, 21}, {1, 22},
- {1, 27},
- {2, 8}, {2, 8}, {2, 8}, {2, 8},
- {2, 11}, {2, 11}, {2, 21},
- {2, 22}, {2, 27}};
-
- Y_UNIT_TEST(Test1) {
- TestWithSaveLoadImpl(input1, expected, true, false);
- }
-
- Y_UNIT_TEST(Test2) {
- TestWithSaveLoadImpl(input1, expected, false, false);
- }
-}
-
-} // namespace NMiniKQL
-} // namespace NKikimr
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+ TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+ }
+
+ TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+ return CreateDeterministicTimeProvider(10000000);
+ }
+
+ TComputationNodeFactory GetAuxCallableFactory() {
+ return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "OneYieldStream") {
+ return new TExternalComputationNode(ctx.Mutables);
+ }
+
+ return GetBuiltinFactory()(callable, ctx);
+ };
+ }
+
+ struct TSetup {
+ TSetup(TScopedAlloc& alloc)
+ : Alloc(alloc)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateRandomProvider();
+ TimeProvider = CreateTimeProvider();
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(),
+ FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
+ return Pattern->Clone(compOpts);
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc& Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+ };
+
+ struct TStreamWithYield : public NUdf::TBoxedValue {
+ TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index)
+ : Items(items)
+ , YieldPos(yieldPos)
+ , Index(index)
+ {}
+
+ private:
+ TUnboxedValueVector Items;
+ ui32 YieldPos;
+ ui32 Index;
+
+ ui32 GetTraverseCount() const override {
+ return 0;
+ }
+
+ NUdf::TUnboxedValue Save() const override {
+ return NUdf::TUnboxedValue::Zero();
+ }
+
+ void Load(const NUdf::TStringRef& state) override {
+ Y_UNUSED(state);
+ }
+
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
+ if (Index >= Items.size()) {
+ return NUdf::EFetchStatus::Finish;
+ }
+ if (Index == YieldPos) {
+ return NUdf::EFetchStatus::Yield;
+ }
+ result = Items[Index++];
+ return NUdf::EFetchStatus::Ok;
+ }
+ };
+
+ THolder<IComputationGraph> BuildGraph(TSetup& setup, const std::vector<std::tuple<ui32, i64, ui32>> items,
+ ui32 yieldPos, ui32 startIndex, bool dataWatermarks) {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto structType = pgmBuilder.NewEmptyStructType();
+ structType = pgmBuilder.NewStructType(structType, "key",
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ structType = pgmBuilder.NewStructType(structType, "time",
+ pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
+ structType = pgmBuilder.NewStructType(structType, "sum",
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key");
+ auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
+ auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
+
+ auto inStreamType = pgmBuilder.NewStreamType(structType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType);
+ auto streamNode = inStream.Build();
+
+ ui64 hop = 10, interval = 30, delay = 20;
+
+ auto pgmReturn = pgmBuilder.MultiHoppingCore(
+ TRuntimeNode(streamNode, false),
+ [&](TRuntimeNode item) { // keyExtractor
+ return pgmBuilder.Member(item, "key");
+ },
+ [&](TRuntimeNode item) { // timeExtractor
+ return pgmBuilder.Member(item, "time");
+ },
+ [&](TRuntimeNode item) { // init
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", pgmBuilder.Member(item, "sum"));
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) { // update
+ auto add = pgmBuilder.AggrAdd(
+ pgmBuilder.Member(item, "sum"),
+ pgmBuilder.Member(state, "sum"));
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", add);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state) { // save
+ return pgmBuilder.Member(state, "sum");
+ },
+ [&](TRuntimeNode savedState) { // load
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", savedState);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
+ auto add = pgmBuilder.AggrAdd(
+ pgmBuilder.Member(state1, "sum"),
+ pgmBuilder.Member(state2, "sum"));
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", add);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish
+ Y_UNUSED(time);
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("key", key);
+ members.emplace_back("sum", pgmBuilder.Member(state, "sum"));
+ return pgmBuilder.NewStruct(members);
+ },
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay
+ pgmBuilder.NewDataLiteral<bool>(dataWatermarks) // dataWatermarks
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+
+ TUnboxedValueVector streamItems;
+ for (size_t i = 0; i < items.size(); ++i) {
+ NUdf::TUnboxedValue* itemsPtr;
+ auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr);
+ itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(std::get<0>(items[i]));
+ itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(std::get<1>(items[i]));
+ itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(std::get<2>(items[i]));
+ streamItems.push_back(std::move(structValues));
+ }
+
+ auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
+ return graph;
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingSaveLoadTest) {
+ void TestWithSaveLoadImpl(
+ const std::vector<std::tuple<ui32, i64, ui32>> input,
+ const std::vector<std::tuple<ui32, ui32>> expected,
+ bool withTraverse,
+ bool dataWatermarks)
+ {
+ TScopedAlloc alloc;
+
+ for (ui32 yieldPos = 0; yieldPos < input.size(); ++yieldPos) {
+ std::vector<std::tuple<ui32, ui32>> result;
+
+ TSetup setup1(alloc);
+ auto graph1 = BuildGraph(setup1, input, yieldPos, 0, dataWatermarks);
+ auto root1 = graph1->GetValue();
+
+ NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root1.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>());
+ }
+ }
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
+
+ TString graphState;
+ if (withTraverse) {
+ SaveGraphState(&root1, 1, 0ULL, graphState);
+ } else {
+ graphState = graph1->SaveGraphState();
+ }
+
+ TSetup setup2(alloc);
+ auto graph2 = BuildGraph(setup2, input, -1, yieldPos, dataWatermarks);
+ NUdf::TUnboxedValue root2;
+ if (withTraverse) {
+ root2 = graph2->GetValue();
+ LoadGraphState(&root2, 1, 0ULL, graphState);
+ } else {
+ graph2->LoadGraphState(graphState);
+ root2 = graph2->GetValue();
+ }
+
+ status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root2.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>());
+ }
+ }
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+
+ auto sortedExpected = expected;
+ std::sort(result.begin(), result.end());
+ std::sort(sortedExpected.begin(), sortedExpected.end());
+ UNIT_ASSERT_EQUAL(result, sortedExpected);
+ }
+ }
+
+ const std::vector<std::tuple<ui32, i64, ui32>> input1 = {
+ // Group; Time; Value
+ {2, 1, 2},
+ {1, 1, 2},
+ {2, 2, 3},
+ {1, 2, 3},
+ {2, 15, 4},
+ {1, 15, 4},
+ {2, 23, 6},
+ {1, 23, 6},
+ {2, 24, 5},
+ {1, 24, 5},
+ {2, 25, 7},
+ {1, 25, 7},
+ {2, 40, 2},
+ {1, 40, 2},
+ {2, 47, 1},
+ {1, 47, 1},
+ {2, 51, 6},
+ {1, 51, 6},
+ {2, 59, 2},
+ {1, 59, 2},
+ {2, 85, 8},
+ {1, 85, 8}
+ };
+
+ const std::vector<std::tuple<ui32, ui32>> expected = {
+ {1, 8}, {1, 8}, {1, 8}, {1, 8},
+ {1, 11}, {1, 11}, {1, 21}, {1, 22},
+ {1, 27},
+ {2, 8}, {2, 8}, {2, 8}, {2, 8},
+ {2, 11}, {2, 11}, {2, 21},
+ {2, 22}, {2, 27}};
+
+ Y_UNIT_TEST(Test1) {
+ TestWithSaveLoadImpl(input1, expected, true, false);
+ }
+
+ Y_UNIT_TEST(Test2) {
+ TestWithSaveLoadImpl(input1, expected, false, false);
+ }
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
index 1a2f47fe734..719ba1f6ea1 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
@@ -7,400 +7,400 @@
#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h>
#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
- struct TInputItem {
- ui32 Key = 0;
- i64 Time = 0;
- ui32 Val = 0;
- };
-
- struct TOutputItem {
- ui32 Key = 0;
- ui32 Val = 0;
- ui64 Time = 0;
-
- constexpr bool operator==(const TOutputItem& rhs) const
- {
- return this->Key == rhs.Key && this->Val == rhs.Val && this->Time == rhs.Time;
- }
- };
-
- struct TOutputGroup {
- TOutputGroup(std::initializer_list<TOutputItem> items) : Items(items) {}
-
- std::vector<TOutputItem> Items;
- };
-
- std::vector<TOutputItem> Ordered(std::vector<TOutputItem> vec) {
- auto res = vec;
- std::sort(res.begin(), res.end(), [](auto l, auto r) {
- return std::make_tuple(l.Key, l.Val, l.Time) < std::make_tuple(r.Key, r.Val, r.Time);
- });
- return res;
- }
-
- IOutputStream &operator<<(IOutputStream &output, std::vector<TOutputItem> items) {
- output << "[";
- for (ui32 i = 0; i < items.size(); ++i) {
- output << "(" << items.at(i).Key << ";" << items.at(i).Val << ";" << items.at(i).Time << ")";
- if (i != items.size() - 1)
- output << ",";
- }
- output << "]";
- return output;
- }
-
- TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
- return CreateDeterministicRandomProvider(1);
- }
-
- TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
- return CreateDeterministicTimeProvider(10000000);
- }
-
- TComputationNodeFactory GetAuxCallableFactory() {
- return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
- if (callable.GetType()->GetName() == "MyStream") {
- return new TExternalComputationNode(ctx.Mutables);
- }
-
- return GetBuiltinFactory()(callable, ctx);
- };
- }
-
- struct TSetup {
- TSetup(TScopedAlloc& alloc)
- : Alloc(alloc)
- {
- FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
- RandomProvider = CreateRandomProvider();
- TimeProvider = CreateTimeProvider();
-
- Env.Reset(new TTypeEnvironment(Alloc));
- PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
- }
-
- THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
- Explorer.Walk(pgm.GetNode(), *Env);
- TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(),
- FunctionRegistry.Get(),
- NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi);
- Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
- TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
- return Pattern->Clone(compOpts);
- }
-
- TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
- TIntrusivePtr<IRandomProvider> RandomProvider;
- TIntrusivePtr<ITimeProvider> TimeProvider;
-
- TScopedAlloc& Alloc;
- THolder<TTypeEnvironment> Env;
- THolder<TProgramBuilder> PgmBuilder;
-
- TExploringNodeVisitor Explorer;
- IComputationPattern::TPtr Pattern;
- };
-
- struct TStream : public NUdf::TBoxedValue {
- TStream(const TUnboxedValueVector& items, std::function<void()> fetchCallback)
- : Items(items)
- , FetchCallback(fetchCallback) {}
-
- private:
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+ struct TInputItem {
+ ui32 Key = 0;
+ i64 Time = 0;
+ ui32 Val = 0;
+ };
+
+ struct TOutputItem {
+ ui32 Key = 0;
+ ui32 Val = 0;
+ ui64 Time = 0;
+
+ constexpr bool operator==(const TOutputItem& rhs) const
+ {
+ return this->Key == rhs.Key && this->Val == rhs.Val && this->Time == rhs.Time;
+ }
+ };
+
+ struct TOutputGroup {
+ TOutputGroup(std::initializer_list<TOutputItem> items) : Items(items) {}
+
+ std::vector<TOutputItem> Items;
+ };
+
+ std::vector<TOutputItem> Ordered(std::vector<TOutputItem> vec) {
+ auto res = vec;
+ std::sort(res.begin(), res.end(), [](auto l, auto r) {
+ return std::make_tuple(l.Key, l.Val, l.Time) < std::make_tuple(r.Key, r.Val, r.Time);
+ });
+ return res;
+ }
+
+ IOutputStream &operator<<(IOutputStream &output, std::vector<TOutputItem> items) {
+ output << "[";
+ for (ui32 i = 0; i < items.size(); ++i) {
+ output << "(" << items.at(i).Key << ";" << items.at(i).Val << ";" << items.at(i).Time << ")";
+ if (i != items.size() - 1)
+ output << ",";
+ }
+ output << "]";
+ return output;
+ }
+
+ TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+ }
+
+ TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+ return CreateDeterministicTimeProvider(10000000);
+ }
+
+ TComputationNodeFactory GetAuxCallableFactory() {
+ return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "MyStream") {
+ return new TExternalComputationNode(ctx.Mutables);
+ }
+
+ return GetBuiltinFactory()(callable, ctx);
+ };
+ }
+
+ struct TSetup {
+ TSetup(TScopedAlloc& alloc)
+ : Alloc(alloc)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateRandomProvider();
+ TimeProvider = CreateTimeProvider();
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(),
+ FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
+ return Pattern->Clone(compOpts);
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc& Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+ };
+
+ struct TStream : public NUdf::TBoxedValue {
+ TStream(const TUnboxedValueVector& items, std::function<void()> fetchCallback)
+ : Items(items)
+ , FetchCallback(fetchCallback) {}
+
+ private:
TUnboxedValueVector Items;
- ui32 Index;
- std::function<void()> FetchCallback;
-
+ ui32 Index;
+ std::function<void()> FetchCallback;
+
NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
- FetchCallback();
- if (Index >= Items.size()) {
- return NUdf::EFetchStatus::Finish;
- }
- result = Items[Index++];
- return NUdf::EFetchStatus::Ok;
- }
- };
-
- THolder<IComputationGraph> BuildGraph(
- TSetup& setup,
- const std::vector<TInputItem> items,
- std::function<void()> fetchCallback,
- bool dataWatermarks,
- ui64 hop = 10,
- ui64 interval = 30,
- ui64 delay = 20)
- {
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto structType = pgmBuilder.NewEmptyStructType();
+ FetchCallback();
+ if (Index >= Items.size()) {
+ return NUdf::EFetchStatus::Finish;
+ }
+ result = Items[Index++];
+ return NUdf::EFetchStatus::Ok;
+ }
+ };
+
+ THolder<IComputationGraph> BuildGraph(
+ TSetup& setup,
+ const std::vector<TInputItem> items,
+ std::function<void()> fetchCallback,
+ bool dataWatermarks,
+ ui64 hop = 10,
+ ui64 interval = 30,
+ ui64 delay = 20)
+ {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto structType = pgmBuilder.NewEmptyStructType();
structType = pgmBuilder.NewStructType(structType, "key",
- pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
structType = pgmBuilder.NewStructType(structType, "time",
- pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
+ pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
structType = pgmBuilder.NewStructType(structType, "sum",
- pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
- auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key");
- auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
- auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
-
- auto inStreamType = pgmBuilder.NewStreamType(structType);
-
- TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "MyStream", inStreamType);
- auto streamNode = inStream.Build();
-
- auto pgmReturn = pgmBuilder.MultiHoppingCore(
- TRuntimeNode(streamNode, false),
- [&](TRuntimeNode item) { // keyExtractor
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key");
+ auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
+ auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
+
+ auto inStreamType = pgmBuilder.NewStreamType(structType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "MyStream", inStreamType);
+ auto streamNode = inStream.Build();
+
+ auto pgmReturn = pgmBuilder.MultiHoppingCore(
+ TRuntimeNode(streamNode, false),
+ [&](TRuntimeNode item) { // keyExtractor
return pgmBuilder.Member(item, "key");
- },
- [&](TRuntimeNode item) { // timeExtractor
+ },
+ [&](TRuntimeNode item) { // timeExtractor
return pgmBuilder.Member(item, "time");
- },
- [&](TRuntimeNode item) { // init
+ },
+ [&](TRuntimeNode item) { // init
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", pgmBuilder.Member(item, "sum"));
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode item, TRuntimeNode state) { // update
- auto add = pgmBuilder.AggrAdd(
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) { // update
+ auto add = pgmBuilder.AggrAdd(
pgmBuilder.Member(item, "sum"),
pgmBuilder.Member(state, "sum"));
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", add);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode state) { // save
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state) { // save
return pgmBuilder.Member(state, "sum");
- },
- [&](TRuntimeNode savedState) { // load
+ },
+ [&](TRuntimeNode savedState) { // load
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", savedState);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
- auto add = pgmBuilder.AggrAdd(
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
+ auto add = pgmBuilder.AggrAdd(
pgmBuilder.Member(state1, "sum"),
pgmBuilder.Member(state2, "sum"));
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("sum", add);
- return pgmBuilder.NewStruct(members);
- },
- [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish
std::vector<std::pair<std::string_view, TRuntimeNode>> members;
members.emplace_back("key", key);
members.emplace_back("sum", pgmBuilder.Member(state, "sum"));
- members.emplace_back("time", time);
- return pgmBuilder.NewStruct(members);
- },
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay
- pgmBuilder.NewDataLiteral<bool>(dataWatermarks) // dataWatermarks
- );
-
- auto graph = setup.BuildGraph(pgmReturn, {streamNode});
-
+ members.emplace_back("time", time);
+ return pgmBuilder.NewStruct(members);
+ },
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay
+ pgmBuilder.NewDataLiteral<bool>(dataWatermarks) // dataWatermarks
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+
TUnboxedValueVector streamItems;
- for (size_t i = 0; i < items.size(); ++i) {
- NUdf::TUnboxedValue* itemsPtr;
- auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr);
- itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(items.at(i).Key);
- itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items.at(i).Time);
- itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items.at(i).Val);
- streamItems.push_back(std::move(structValues));
- }
-
- auto streamValue = NUdf::TUnboxedValuePod(new TStream(streamItems, fetchCallback));
- graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
- return graph;
- }
-}
-
-Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingTest) {
- void TestImpl(
- const std::vector<TInputItem> input,
- const std::vector<TOutputGroup> expected,
- bool dataWatermarks,
- ui64 hop = 10,
- ui64 interval = 30,
- ui64 delay = 20)
- {
- TScopedAlloc alloc;
- TSetup setup1(alloc);
-
- ui32 curGroupId = 0;
- std::vector<TOutputItem> curResult;
-
- auto check = [&curResult, &curGroupId, &expected]() {
- auto expectedItems = Ordered(expected.at(curGroupId).Items);
- curResult = Ordered(curResult);
- UNIT_ASSERT_EQUAL_C(curResult, expectedItems, "curGroup: " << curGroupId << " actual: " << curResult << " expected: " << expectedItems);
- curGroupId++;
- curResult.clear();
- };
-
- auto graph1 = BuildGraph(setup1, input, check, dataWatermarks, hop, interval, delay);
-
- auto root1 = graph1->GetValue();
-
- NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
- while (status == NUdf::EFetchStatus::Ok) {
- NUdf::TUnboxedValue val;
- status = root1.Fetch(val);
- if (status == NUdf::EFetchStatus::Ok) {
- curResult.emplace_back(TOutputItem{val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>(), val.GetElement(2).Get<ui64>()});
+ for (size_t i = 0; i < items.size(); ++i) {
+ NUdf::TUnboxedValue* itemsPtr;
+ auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr);
+ itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(items.at(i).Key);
+ itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items.at(i).Time);
+ itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items.at(i).Val);
+ streamItems.push_back(std::move(structValues));
+ }
+
+ auto streamValue = NUdf::TUnboxedValuePod(new TStream(streamItems, fetchCallback));
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
+ return graph;
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingTest) {
+ void TestImpl(
+ const std::vector<TInputItem> input,
+ const std::vector<TOutputGroup> expected,
+ bool dataWatermarks,
+ ui64 hop = 10,
+ ui64 interval = 30,
+ ui64 delay = 20)
+ {
+ TScopedAlloc alloc;
+ TSetup setup1(alloc);
+
+ ui32 curGroupId = 0;
+ std::vector<TOutputItem> curResult;
+
+ auto check = [&curResult, &curGroupId, &expected]() {
+ auto expectedItems = Ordered(expected.at(curGroupId).Items);
+ curResult = Ordered(curResult);
+ UNIT_ASSERT_EQUAL_C(curResult, expectedItems, "curGroup: " << curGroupId << " actual: " << curResult << " expected: " << expectedItems);
+ curGroupId++;
+ curResult.clear();
+ };
+
+ auto graph1 = BuildGraph(setup1, input, check, dataWatermarks, hop, interval, delay);
+
+ auto root1 = graph1->GetValue();
+
+ NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root1.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ curResult.emplace_back(TOutputItem{val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>(), val.GetElement(2).Get<ui64>()});
}
- }
-
- check();
- // TODO: some problem with parallel run
- //UNIT_ASSERT_EQUAL_C(curGroupId, expected.size(), "1: " << curGroupId << " 2: " << expected.size());
- }
-
- Y_UNIT_TEST(TestDataWatermarks) {
- const std::vector<TInputItem> input = {
- // Group; Time; Value
- {1, 101, 2},
- {2, 101, 2},
- {1, 111, 3},
- {2, 140, 5},
- {2, 160, 1}
- };
- const std::vector<TOutputGroup> expected = {
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({{1, 2, 110}, {1, 5, 120}, {2, 2, 110}, {2, 2, 120}}),
- TOutputGroup({{2, 2, 130}, {1, 5, 130}, {1, 3, 140}}),
- TOutputGroup({{2, 5, 150}, {2, 5, 160}, {2, 6, 170}, {2, 1, 180}, {2, 1, 190}}),
- };
- TestImpl(input, expected, true);
- }
-
- Y_UNIT_TEST(TestValidness1) {
- const std::vector<TInputItem> input1 = {
- // Group; Time; Value
- {1, 101, 2},
- {2, 101, 2},
- {1, 111, 3},
- {2, 140, 5},
- {2, 160, 1}
- };
-
- const std::vector<TOutputGroup> expected = {
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({{2, 2, 110}, {2, 2, 120}}),
- TOutputGroup({{2, 2, 130}}),
- TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}, {2, 5, 150},
- {2, 5, 160}, {2, 6, 170}, {2, 1, 190}, {2, 1, 180}}),
- };
- TestImpl(input1, expected, false);
- }
-
- Y_UNIT_TEST(TestValidness2) {
- const std::vector<TInputItem> input = {
- // Group; Time; Value
- {2, 101, 2}, {1, 101, 2}, {2, 102, 3}, {1, 102, 3}, {2, 115, 4},
- {1, 115, 4}, {2, 123, 6}, {1, 123, 6}, {2, 124, 5}, {1, 124, 5},
- {2, 125, 7}, {1, 125, 7}, {2, 140, 2}, {1, 140, 2}, {2, 147, 1},
- {1, 147, 1}, {2, 151, 6}, {1, 151, 6}, {2, 159, 2}, {1, 159, 2},
- {2, 185, 8}, {1, 185, 8}
- };
- const std::vector<TOutputGroup> expected = {
- TOutputGroup({}),
- TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({{1, 5, 110}, {1, 9, 120}, {2, 5, 110}, {2, 9, 120}}),
- TOutputGroup({}),
- TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({{2, 27, 130}, {1, 27, 130}}),
- TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({{2, 22, 140}, {2, 21, 150}, {2, 11, 160}, {1, 22, 140}, {1, 21, 150}, {1, 11, 160}}),
- TOutputGroup({}),
- TOutputGroup({{1, 11, 170}, {1, 8, 180}, {1, 8, 190}, {1, 8, 200}, {1, 8, 210}, {2, 11, 170},
- {2, 8, 180}, {2, 8, 190}, {2, 8, 200}, {2, 8, 210}}),
- };
-
- TestImpl(input, expected, true);
- }
-
- Y_UNIT_TEST(TestValidness3) {
- const std::vector<TInputItem> input = {
- // Group; Time; Value
- {1, 105, 1}, {1, 107, 4}, {2, 106, 3}, {1, 111, 7}, {1, 117, 3},
- {2, 110, 2}, {1, 108, 9}, {1, 121, 4}, {2, 107, 2}, {2, 141, 5},
- {1, 141, 10}
- };
- const std::vector<TOutputGroup> expected = {
- TOutputGroup({}),
- TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({{1, 14, 110}, {2, 3, 110}}),
- TOutputGroup({}),
- TOutputGroup({{2, 7, 115}, {2, 2, 120}, {1, 21, 115}, {1, 10, 120}, {1, 7, 125}, {1, 4, 130}}),
- TOutputGroup({}),
- TOutputGroup({{1, 10, 145}, {1, 10, 150}, {2, 5, 145}, {2, 5, 150}})
- };
-
- TestImpl(input, expected, true, 5, 10, 10);
+ }
+
+ check();
+ // TODO: some problem with parallel run
+ //UNIT_ASSERT_EQUAL_C(curGroupId, expected.size(), "1: " << curGroupId << " 2: " << expected.size());
+ }
+
+ Y_UNIT_TEST(TestDataWatermarks) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 2},
+ {2, 101, 2},
+ {1, 111, 3},
+ {2, 140, 5},
+ {2, 160, 1}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 2, 110}, {1, 5, 120}, {2, 2, 110}, {2, 2, 120}}),
+ TOutputGroup({{2, 2, 130}, {1, 5, 130}, {1, 3, 140}}),
+ TOutputGroup({{2, 5, 150}, {2, 5, 160}, {2, 6, 170}, {2, 1, 180}, {2, 1, 190}}),
+ };
+ TestImpl(input, expected, true);
+ }
+
+ Y_UNIT_TEST(TestValidness1) {
+ const std::vector<TInputItem> input1 = {
+ // Group; Time; Value
+ {1, 101, 2},
+ {2, 101, 2},
+ {1, 111, 3},
+ {2, 140, 5},
+ {2, 160, 1}
+ };
+
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{2, 2, 110}, {2, 2, 120}}),
+ TOutputGroup({{2, 2, 130}}),
+ TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}, {2, 5, 150},
+ {2, 5, 160}, {2, 6, 170}, {2, 1, 190}, {2, 1, 180}}),
+ };
+ TestImpl(input1, expected, false);
+ }
+
+ Y_UNIT_TEST(TestValidness2) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {2, 101, 2}, {1, 101, 2}, {2, 102, 3}, {1, 102, 3}, {2, 115, 4},
+ {1, 115, 4}, {2, 123, 6}, {1, 123, 6}, {2, 124, 5}, {1, 124, 5},
+ {2, 125, 7}, {1, 125, 7}, {2, 140, 2}, {1, 140, 2}, {2, 147, 1},
+ {1, 147, 1}, {2, 151, 6}, {1, 151, 6}, {2, 159, 2}, {1, 159, 2},
+ {2, 185, 8}, {1, 185, 8}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{1, 5, 110}, {1, 9, 120}, {2, 5, 110}, {2, 9, 120}}),
+ TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{2, 27, 130}, {1, 27, 130}}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{2, 22, 140}, {2, 21, 150}, {2, 11, 160}, {1, 22, 140}, {1, 21, 150}, {1, 11, 160}}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 11, 170}, {1, 8, 180}, {1, 8, 190}, {1, 8, 200}, {1, 8, 210}, {2, 11, 170},
+ {2, 8, 180}, {2, 8, 190}, {2, 8, 200}, {2, 8, 210}}),
+ };
+
+ TestImpl(input, expected, true);
+ }
+
+ Y_UNIT_TEST(TestValidness3) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 105, 1}, {1, 107, 4}, {2, 106, 3}, {1, 111, 7}, {1, 117, 3},
+ {2, 110, 2}, {1, 108, 9}, {1, 121, 4}, {2, 107, 2}, {2, 141, 5},
+ {1, 141, 10}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{1, 14, 110}, {2, 3, 110}}),
+ TOutputGroup({}),
+ TOutputGroup({{2, 7, 115}, {2, 2, 120}, {1, 21, 115}, {1, 10, 120}, {1, 7, 125}, {1, 4, 130}}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 10, 145}, {1, 10, 150}, {2, 5, 145}, {2, 5, 150}})
+ };
+
+ TestImpl(input, expected, true, 5, 10, 10);
}
- Y_UNIT_TEST(TestDelay) {
- const std::vector<TInputItem> input = {
- // Group; Time; Value
- {1, 101, 3}, {1, 111, 5}, {1, 120, 7}, {1, 80, 9}, {1, 79, 11}
- };
- const std::vector<TOutputGroup> expected = {
- TOutputGroup({}),
- TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({}), TOutputGroup({}),
- TOutputGroup({{1, 12, 110}, {1, 8, 120}, {1, 15, 130}, {1, 12, 140}, {1, 7, 150}})
- };
-
- TestImpl(input, expected, false);
+ Y_UNIT_TEST(TestDelay) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 3}, {1, 111, 5}, {1, 120, 7}, {1, 80, 9}, {1, 79, 11}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{1, 12, 110}, {1, 8, 120}, {1, 15, 130}, {1, 12, 140}, {1, 7, 150}})
+ };
+
+ TestImpl(input, expected, false);
+ }
+
+ Y_UNIT_TEST(TestWindowsBeforeFirstElement) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 2}, {1, 111, 3}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}})
+ };
+
+ TestImpl(input, expected, false);
}
-
- Y_UNIT_TEST(TestWindowsBeforeFirstElement) {
- const std::vector<TInputItem> input = {
- // Group; Time; Value
- {1, 101, 2}, {1, 111, 3}
- };
- const std::vector<TOutputGroup> expected = {
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}})
- };
-
- TestImpl(input, expected, false);
- }
-
- Y_UNIT_TEST(TestSubzeroValues) {
- const std::vector<TInputItem> input = {
- // Group; Time; Value
- {1, 1, 2}
- };
- const std::vector<TOutputGroup> expected = {
- TOutputGroup({}),
- TOutputGroup({}),
- TOutputGroup({{1, 2, 30}}),
- };
-
- TestImpl(input, expected, false);
- }
-}
-
-} // namespace NMiniKQL
-} // namespace NKikimr
+
+ Y_UNIT_TEST(TestSubzeroValues) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 1, 2}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 2, 30}}),
+ };
+
+ TestImpl(input, expected, false);
+ }
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/ya.make b/ydb/library/yql/minikql/comp_nodes/ut/ya.make
index 38243bc1d37..dffda1318f3 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/ya.make
+++ b/ydb/library/yql/minikql/comp_nodes/ut/ya.make
@@ -27,8 +27,8 @@ SRCS(
mkql_chopper_ut.cpp
mkql_filters_ut.cpp
mkql_flatmap_ut.cpp
- mkql_multihopping_saveload_ut.cpp
- mkql_multihopping_ut.cpp
+ mkql_multihopping_saveload_ut.cpp
+ mkql_multihopping_ut.cpp
mkql_multimap_ut.cpp
mkql_fold_ut.cpp
mkql_heap_ut.cpp
diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make
index b4b5ee777bb..55838c40123 100644
--- a/ydb/library/yql/minikql/comp_nodes/ya.make
+++ b/ydb/library/yql/minikql/comp_nodes/ya.make
@@ -112,8 +112,8 @@ SRCS(
mkql_map.h
mkql_map_join.cpp
mkql_map_join.h
- mkql_multihopping.cpp
- mkql_multihopping.h
+ mkql_multihopping.cpp
+ mkql_multihopping.h
mkql_multimap.cpp
mkql_multimap.h
mkql_next_value.cpp
diff --git a/ydb/library/yql/minikql/mkql_alloc.h b/ydb/library/yql/minikql/mkql_alloc.h
index ee254585394..5e864cdc4c8 100644
--- a/ydb/library/yql/minikql/mkql_alloc.h
+++ b/ydb/library/yql/minikql/mkql_alloc.h
@@ -120,7 +120,7 @@ public:
void InvalidateMemInfo() { MyState_.InvalidateMemInfo(); }
bool IsAttached() const { return AttachedCount_ > 0; }
-
+
private:
TAllocState MyState_;
size_t AttachedCount_ = 0;
diff --git a/ydb/library/yql/minikql/mkql_node.h b/ydb/library/yql/minikql/mkql_node.h
index e7f9c9ec60d..ee74455ea69 100644
--- a/ydb/library/yql/minikql/mkql_node.h
+++ b/ydb/library/yql/minikql/mkql_node.h
@@ -462,8 +462,8 @@ public:
return Guard(Alloc);
}
- TScopedAlloc& GetAllocator() const { return Alloc; }
-
+ TScopedAlloc& GetAllocator() const { return Alloc; }
+
const NUdf::TStringValue& NewString(ui32 size) const {
Strings.emplace(size);
return Strings.top();
diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp
index de7b56c0de9..47ae6e4c8fc 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.cpp
+++ b/ydb/library/yql/minikql/mkql_program_builder.cpp
@@ -4660,89 +4660,89 @@ TRuntimeNode TProgramBuilder::HoppingCore(TRuntimeNode list,
return TRuntimeNode(callableBuilder.Build(), false);
}
-TRuntimeNode TProgramBuilder::MultiHoppingCore(TRuntimeNode list,
- const TUnaryLambda& keyExtractor,
- const TUnaryLambda& timeExtractor,
- const TUnaryLambda& init,
- const TBinaryLambda& update,
- const TUnaryLambda& save,
- const TUnaryLambda& load,
- const TBinaryLambda& merge,
- const TTernaryLambda& finish,
- TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay, TRuntimeNode dataWatermarks)
-{
- if constexpr (RuntimeVersion < 22U) {
- THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__;
- }
-
- auto streamType = AS_TYPE(TStreamType, list);
- auto itemType = AS_TYPE(TStructType, streamType->GetItemType());
- auto timestampType = TOptionalType::Create(TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, Env), Env);
-
- TRuntimeNode itemArg = Arg(itemType);
-
- auto keyExtract = keyExtractor(itemArg);
- auto keyType = keyExtract.GetStaticType();
- TRuntimeNode keyArg = Arg(keyType);
-
- auto outTime = timeExtractor(itemArg);
- auto outStateInit = init(itemArg);
-
- auto stateType = outStateInit.GetStaticType();
- TRuntimeNode stateArg = Arg(stateType);
-
- auto outStateUpdate = update(itemArg, stateArg);
-
- auto hasSaveLoad = (bool)save;
- TRuntimeNode saveArg, outSave, loadArg, outLoad;
- if (hasSaveLoad) {
- saveArg = Arg(stateType);
- outSave = save(saveArg);
-
- loadArg = Arg(outSave.GetStaticType());
- outLoad = load(loadArg);
-
- MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*stateType), "Loaded type is changed by the load handler");
- } else {
- saveArg = outSave = loadArg = outLoad = NewVoid();
- }
-
- TRuntimeNode state2Arg = Arg(stateType);
- TRuntimeNode timeArg = Arg(timestampType);
-
- auto outStateMerge = merge(stateArg, state2Arg);
- auto outItemFinish = finish(keyArg, stateArg, timeArg);
-
- auto finishType = outItemFinish.GetStaticType();
- MKQL_ENSURE(finishType->IsStruct(), "Expected struct type as finish lambda output");
-
- auto resultType = TStreamType::Create(outItemFinish.GetStaticType(), Env);
-
- TCallableBuilder callableBuilder(Env, __func__, resultType);
- callableBuilder.Add(list);
- callableBuilder.Add(itemArg);
- callableBuilder.Add(keyArg);
- callableBuilder.Add(stateArg);
- callableBuilder.Add(state2Arg);
- callableBuilder.Add(timeArg);
- callableBuilder.Add(saveArg);
- callableBuilder.Add(loadArg);
- callableBuilder.Add(keyExtract);
- callableBuilder.Add(outTime);
- callableBuilder.Add(outStateInit);
- callableBuilder.Add(outStateUpdate);
- callableBuilder.Add(outSave);
- callableBuilder.Add(outLoad);
- callableBuilder.Add(outStateMerge);
- callableBuilder.Add(outItemFinish);
- callableBuilder.Add(hop);
- callableBuilder.Add(interval);
- callableBuilder.Add(delay);
- callableBuilder.Add(dataWatermarks);
-
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
+TRuntimeNode TProgramBuilder::MultiHoppingCore(TRuntimeNode list,
+ const TUnaryLambda& keyExtractor,
+ const TUnaryLambda& timeExtractor,
+ const TUnaryLambda& init,
+ const TBinaryLambda& update,
+ const TUnaryLambda& save,
+ const TUnaryLambda& load,
+ const TBinaryLambda& merge,
+ const TTernaryLambda& finish,
+ TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay, TRuntimeNode dataWatermarks)
+{
+ if constexpr (RuntimeVersion < 22U) {
+ THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__;
+ }
+
+ auto streamType = AS_TYPE(TStreamType, list);
+ auto itemType = AS_TYPE(TStructType, streamType->GetItemType());
+ auto timestampType = TOptionalType::Create(TDataType::Create(NUdf::TDataType<NUdf::TTimestamp>::Id, Env), Env);
+
+ TRuntimeNode itemArg = Arg(itemType);
+
+ auto keyExtract = keyExtractor(itemArg);
+ auto keyType = keyExtract.GetStaticType();
+ TRuntimeNode keyArg = Arg(keyType);
+
+ auto outTime = timeExtractor(itemArg);
+ auto outStateInit = init(itemArg);
+
+ auto stateType = outStateInit.GetStaticType();
+ TRuntimeNode stateArg = Arg(stateType);
+
+ auto outStateUpdate = update(itemArg, stateArg);
+
+ auto hasSaveLoad = (bool)save;
+ TRuntimeNode saveArg, outSave, loadArg, outLoad;
+ if (hasSaveLoad) {
+ saveArg = Arg(stateType);
+ outSave = save(saveArg);
+
+ loadArg = Arg(outSave.GetStaticType());
+ outLoad = load(loadArg);
+
+ MKQL_ENSURE(outLoad.GetStaticType()->IsSameType(*stateType), "Loaded type is changed by the load handler");
+ } else {
+ saveArg = outSave = loadArg = outLoad = NewVoid();
+ }
+
+ TRuntimeNode state2Arg = Arg(stateType);
+ TRuntimeNode timeArg = Arg(timestampType);
+
+ auto outStateMerge = merge(stateArg, state2Arg);
+ auto outItemFinish = finish(keyArg, stateArg, timeArg);
+
+ auto finishType = outItemFinish.GetStaticType();
+ MKQL_ENSURE(finishType->IsStruct(), "Expected struct type as finish lambda output");
+
+ auto resultType = TStreamType::Create(outItemFinish.GetStaticType(), Env);
+
+ TCallableBuilder callableBuilder(Env, __func__, resultType);
+ callableBuilder.Add(list);
+ callableBuilder.Add(itemArg);
+ callableBuilder.Add(keyArg);
+ callableBuilder.Add(stateArg);
+ callableBuilder.Add(state2Arg);
+ callableBuilder.Add(timeArg);
+ callableBuilder.Add(saveArg);
+ callableBuilder.Add(loadArg);
+ callableBuilder.Add(keyExtract);
+ callableBuilder.Add(outTime);
+ callableBuilder.Add(outStateInit);
+ callableBuilder.Add(outStateUpdate);
+ callableBuilder.Add(outSave);
+ callableBuilder.Add(outLoad);
+ callableBuilder.Add(outStateMerge);
+ callableBuilder.Add(outItemFinish);
+ callableBuilder.Add(hop);
+ callableBuilder.Add(interval);
+ callableBuilder.Add(delay);
+ callableBuilder.Add(dataWatermarks);
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
TRuntimeNode TProgramBuilder::Default(TType* type) {
bool isOptional;
const auto targetType = UnpackOptionalData(type, isOptional);
diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h
index fcaf39d5e5e..10f1ad7ccf1 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.h
+++ b/ydb/library/yql/minikql/mkql_program_builder.h
@@ -425,16 +425,16 @@ public:
const TBinaryLambda& merge,
const TBinaryLambda& finish,
TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay);
- TRuntimeNode MultiHoppingCore(TRuntimeNode list,
- const TUnaryLambda& keyExtractor,
- const TUnaryLambda& timeExtractor,
- const TUnaryLambda& init,
- const TBinaryLambda& update,
- const TUnaryLambda& save,
- const TUnaryLambda& load,
- const TBinaryLambda& merge,
- const TTernaryLambda& finish,
- TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay, TRuntimeNode dataWatermarks);
+ TRuntimeNode MultiHoppingCore(TRuntimeNode list,
+ const TUnaryLambda& keyExtractor,
+ const TUnaryLambda& timeExtractor,
+ const TUnaryLambda& init,
+ const TBinaryLambda& update,
+ const TUnaryLambda& save,
+ const TUnaryLambda& load,
+ const TBinaryLambda& merge,
+ const TTernaryLambda& finish,
+ TRuntimeNode hop, TRuntimeNode interval, TRuntimeNode delay, TRuntimeNode dataWatermarks);
TRuntimeNode Chopper(TRuntimeNode flow, const TUnaryLambda& keyExtractor, const TBinaryLambda& groupSwitch, const TBinaryLambda& groupHandler);
diff --git a/ydb/library/yql/minikql/ut/ya.make b/ydb/library/yql/minikql/ut/ya.make
index 74000c251c9..098a77d0c60 100644
--- a/ydb/library/yql/minikql/ut/ya.make
+++ b/ydb/library/yql/minikql/ut/ya.make
@@ -28,7 +28,7 @@ SRCS(
mkql_type_ops_ut.cpp
mkql_string_util_ut.cpp
pack_num_ut.cpp
- watermark_tracker_ut.cpp
+ watermark_tracker_ut.cpp
)
PEERDIR(
diff --git a/ydb/library/yql/minikql/watermark_tracker.cpp b/ydb/library/yql/minikql/watermark_tracker.cpp
index b359c513b97..2e08bb08e76 100644
--- a/ydb/library/yql/minikql/watermark_tracker.cpp
+++ b/ydb/library/yql/minikql/watermark_tracker.cpp
@@ -1,39 +1,39 @@
-#include "watermark_tracker.h"
-
-#include<util/system/yassert.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-TWatermarkTracker::TWatermarkTracker(
- ui64 delay,
- ui64 granularity)
- : Delay(delay)
- , Granularity(granularity)
-{
- Y_VERIFY(granularity > 0);
-}
-
-std::optional<ui64> TWatermarkTracker::HandleNextEventTime(ui64 ts) {
- if (Y_UNLIKELY(ts >= NextEventWithWatermark)) {
- NextEventWithWatermark = CalcNextEventWithWatermark(ts);
- return CalcLastWatermark();
- }
-
- return std::nullopt;
-}
-
-ui64 TWatermarkTracker::CalcNextEventWithWatermark(ui64 ts) {
- return ts + Granularity - (ts - Delay) % Granularity;
-}
-
-std::optional<ui64> TWatermarkTracker::CalcLastWatermark() {
- if (Y_UNLIKELY(Delay + Granularity > NextEventWithWatermark)) {
- // Protect from negative values
- return std::nullopt;
- }
- return NextEventWithWatermark - Delay - Granularity;
-}
-
-} // NMiniKQL
-} // NKikimr
+#include "watermark_tracker.h"
+
+#include<util/system/yassert.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+TWatermarkTracker::TWatermarkTracker(
+ ui64 delay,
+ ui64 granularity)
+ : Delay(delay)
+ , Granularity(granularity)
+{
+ Y_VERIFY(granularity > 0);
+}
+
+std::optional<ui64> TWatermarkTracker::HandleNextEventTime(ui64 ts) {
+ if (Y_UNLIKELY(ts >= NextEventWithWatermark)) {
+ NextEventWithWatermark = CalcNextEventWithWatermark(ts);
+ return CalcLastWatermark();
+ }
+
+ return std::nullopt;
+}
+
+ui64 TWatermarkTracker::CalcNextEventWithWatermark(ui64 ts) {
+ return ts + Granularity - (ts - Delay) % Granularity;
+}
+
+std::optional<ui64> TWatermarkTracker::CalcLastWatermark() {
+ if (Y_UNLIKELY(Delay + Granularity > NextEventWithWatermark)) {
+ // Protect from negative values
+ return std::nullopt;
+ }
+ return NextEventWithWatermark - Delay - Granularity;
+}
+
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/watermark_tracker.h b/ydb/library/yql/minikql/watermark_tracker.h
index 7926c4b583d..aa06f614f4e 100644
--- a/ydb/library/yql/minikql/watermark_tracker.h
+++ b/ydb/library/yql/minikql/watermark_tracker.h
@@ -1,25 +1,25 @@
-#pragma once
-
-#include <optional>
-#include <util/system/types.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-class TWatermarkTracker {
-public:
- TWatermarkTracker(ui64 lag, ui64 granularity);
- std::optional<ui64> HandleNextEventTime(ui64 ts);
-
-private:
- ui64 CalcNextEventWithWatermark(ui64 ts);
- std::optional<ui64> CalcLastWatermark();
-
-private:
- ui64 NextEventWithWatermark = 0;
- const ui64 Delay;
- const ui64 Granularity;
-};
-
-} // NMiniKQL
-} // NKikimr
+#pragma once
+
+#include <optional>
+#include <util/system/types.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+class TWatermarkTracker {
+public:
+ TWatermarkTracker(ui64 lag, ui64 granularity);
+ std::optional<ui64> HandleNextEventTime(ui64 ts);
+
+private:
+ ui64 CalcNextEventWithWatermark(ui64 ts);
+ std::optional<ui64> CalcLastWatermark();
+
+private:
+ ui64 NextEventWithWatermark = 0;
+ const ui64 Delay;
+ const ui64 Granularity;
+};
+
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/watermark_tracker_ut.cpp b/ydb/library/yql/minikql/watermark_tracker_ut.cpp
index 33cc3492458..ab2864caa21 100644
--- a/ydb/library/yql/minikql/watermark_tracker_ut.cpp
+++ b/ydb/library/yql/minikql/watermark_tracker_ut.cpp
@@ -1,26 +1,26 @@
-#include "watermark_tracker.h"
-
-#include <util/system/types.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-Y_UNIT_TEST_SUITE(TWatermarkTrackerTest) {
- Y_UNIT_TEST(WhenNotGranularShouldNotMoveWatermark) {
- TWatermarkTracker tracker(15, 10);
- UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(25), 10);
- UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(26), std::nullopt);
- UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(30), std::nullopt);
- UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(34), std::nullopt);
- }
-
- Y_UNIT_TEST(WhenGranularShouldMoveWatermark) {
- TWatermarkTracker tracker(15, 10);
- UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(35), 20);
- UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(45), 30);
- }
-}
-
-} // NMiniKQL
-} // NKikimr \ No newline at end of file
+#include "watermark_tracker.h"
+
+#include <util/system/types.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TWatermarkTrackerTest) {
+ Y_UNIT_TEST(WhenNotGranularShouldNotMoveWatermark) {
+ TWatermarkTracker tracker(15, 10);
+ UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(25), 10);
+ UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(26), std::nullopt);
+ UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(30), std::nullopt);
+ UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(34), std::nullopt);
+ }
+
+ Y_UNIT_TEST(WhenGranularShouldMoveWatermark) {
+ TWatermarkTracker tracker(15, 10);
+ UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(35), 20);
+ UNIT_ASSERT_EQUAL(tracker.HandleNextEventTime(45), 30);
+ }
+}
+
+} // NMiniKQL
+} // NKikimr \ No newline at end of file
diff --git a/ydb/library/yql/minikql/ya.make b/ydb/library/yql/minikql/ya.make
index 2df85e818c0..c4b608128d8 100644
--- a/ydb/library/yql/minikql/ya.make
+++ b/ydb/library/yql/minikql/ya.make
@@ -48,8 +48,8 @@ SRCS(
mkql_unboxed_value_stream.h
primes.cpp
primes.h
- watermark_tracker.cpp
- watermark_tracker.h
+ watermark_tracker.cpp
+ watermark_tracker.h
)
PEERDIR(
diff --git a/ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp b/ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp
index 94adf7ba30a..88aea459c33 100644
--- a/ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp
+++ b/ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.cpp
@@ -2,13 +2,13 @@
#include "yql_ch_read_actor.h"
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h>
-
+
namespace NYql::NDq {
void RegisterClickHouseReadActorFactory(TDqSourceFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IHTTPGateway::TPtr gateway) {
factory.Register<NCH::TSource>("ClickHouseSource",
- [credentialsFactory, gateway](NCH::TSource&& settings, IDqSourceActorFactory::TArguments&& args) {
- return CreateClickHouseReadActor(gateway, std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, credentialsFactory);
+ [credentialsFactory, gateway](NCH::TSource&& settings, IDqSourceActorFactory::TArguments&& args) {
+ return CreateClickHouseReadActor(gateway, std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, credentialsFactory);
});
}
diff --git a/ydb/library/yql/providers/common/mkql/parser.cpp b/ydb/library/yql/providers/common/mkql/parser.cpp
index 1c0fda5860d..bfdc26f5f85 100644
--- a/ydb/library/yql/providers/common/mkql/parser.cpp
+++ b/ydb/library/yql/providers/common/mkql/parser.cpp
@@ -1,105 +1,105 @@
-#include "parser.h"
-#include "util/generic/maybe.h"
-
-#include <functional>
-#include <string_view>
+#include "parser.h"
+#include "util/generic/maybe.h"
+
+#include <functional>
+#include <string_view>
#include <ydb/library/yql/minikql/defs.h>
-
-#include <library/cpp/json/json_writer.h>
-
-namespace NYql {
-
-using namespace NKikimr::NMiniKQL;
-using namespace NNodes;
-
-namespace {
-
-const TExprNode& GetFormat(const TExprNode& settings) {
- for (auto i = 0U; i < settings.ChildrenSize(); ++i) {
- const auto& child = *settings.Child(i);
- if (child.Head().IsAtom("format"))
- return child.Tail();
- }
- THROW yexception() << "Unknown format.";
-}
-
-std::array<TString, 2U> GetSettings(const TExprNode& settings) {
- for (auto i = 0U; i < settings.ChildrenSize(); ++i) {
- const auto& child = *settings.Child(i);
- if (child.Head().IsAtom("settings")) {
- if (child.Tail().IsList()) {
- TString json, compression;
- TStringOutput stream(json);
- NJson::TJsonWriter writer(&stream, NJson::TJsonWriterConfig());
- writer.OpenMap();
- child.Tail().ForEachChild([&writer, &compression](const TExprNode& pair) {
- if (pair.Head().IsAtom("compression") && pair.Tail().IsCallable({"String", "Utf8"}))
- if (const auto& comp = pair.Tail().Head().Content(); !comp.empty() && std::isupper(comp.front()))
- compression = comp;
- else {
- writer.WriteKey(pair.Head().Content());
- writer.Write(comp);
- }
- else if (pair.Head().IsAtom() && pair.Tail().IsCallable({"Bool", "Float", "Double", "Int8", "Uint8", "Int16", "Uint16", "Int32", "Uint32", "Int64", "Uint64", "String", "Utf8"})) {
- writer.WriteKey(pair.Head().Content());
- if (const auto& type = pair.Tail().Content(); "Bool" == type)
- writer.Write(FromString<bool>(pair.Tail().Head().Content()));
- else if ("Float" == type)
- writer.Write(FromString<float>(pair.Tail().Head().Content()));
- else if ("Double" == type)
- writer.Write(FromString<double>(pair.Tail().Head().Content()));
- else if ("Int8" == type)
- writer.Write(FromString<i8>(pair.Tail().Head().Content()));
- else if ("Uint8" == type)
- writer.Write(FromString<ui8>(pair.Tail().Head().Content()));
- else if ("Int16" == type)
- writer.Write(FromString<i16>(pair.Tail().Head().Content()));
- else if ("Uint16" == type)
- writer.Write(FromString<ui16>(pair.Tail().Head().Content()));
- else if ("Int32" == type)
- writer.Write(FromString<i32>(pair.Tail().Head().Content()));
- else if ("Uint32" == type)
- writer.Write(FromString<ui32>(pair.Tail().Head().Content()));
- else if ("Int64" == type)
- writer.Write(FromString<i64>(pair.Tail().Head().Content()));
- else if ("Uint64" == type)
- writer.Write(FromString<ui64>(pair.Tail().Head().Content()));
- else
- writer.Write(pair.Tail().Head().Content());
-
- }
- });
- writer.CloseMap();
- writer.Flush();
- if (json == "{}")
- json.clear();
- return {std::move(json), std::move(compression)};
- }
- }
- }
- return {TString(), TString()};
-}
-}
-
-TRuntimeNode BuildParseCall(
- TRuntimeNode input,
- const std::string_view& format,
- const std::string_view& compression,
- TType* inputItemType,
- TType* outputItemType,
- NCommon::TMkqlBuildContext& ctx)
-{
- if (!compression.empty()) {
- input = ctx.ProgramBuilder.Map(input, [&ctx, &compression](TRuntimeNode item) {
- return ctx.ProgramBuilder.Apply(ctx.ProgramBuilder.Udf(std::string("Decompress.") += compression), {item});
- });
- }
-
- const auto structType = static_cast<const TStructType*>(outputItemType);
- if (format == "raw") {
- MKQL_ENSURE(1U == structType->GetMembersCount(), "Expected single column.");
- bool isOptional;
- const auto schemeType = UnpackOptionalData(structType->GetMemberType(0U), isOptional)->GetSchemeType();
+
+#include <library/cpp/json/json_writer.h>
+
+namespace NYql {
+
+using namespace NKikimr::NMiniKQL;
+using namespace NNodes;
+
+namespace {
+
+const TExprNode& GetFormat(const TExprNode& settings) {
+ for (auto i = 0U; i < settings.ChildrenSize(); ++i) {
+ const auto& child = *settings.Child(i);
+ if (child.Head().IsAtom("format"))
+ return child.Tail();
+ }
+ THROW yexception() << "Unknown format.";
+}
+
+std::array<TString, 2U> GetSettings(const TExprNode& settings) {
+ for (auto i = 0U; i < settings.ChildrenSize(); ++i) {
+ const auto& child = *settings.Child(i);
+ if (child.Head().IsAtom("settings")) {
+ if (child.Tail().IsList()) {
+ TString json, compression;
+ TStringOutput stream(json);
+ NJson::TJsonWriter writer(&stream, NJson::TJsonWriterConfig());
+ writer.OpenMap();
+ child.Tail().ForEachChild([&writer, &compression](const TExprNode& pair) {
+ if (pair.Head().IsAtom("compression") && pair.Tail().IsCallable({"String", "Utf8"}))
+ if (const auto& comp = pair.Tail().Head().Content(); !comp.empty() && std::isupper(comp.front()))
+ compression = comp;
+ else {
+ writer.WriteKey(pair.Head().Content());
+ writer.Write(comp);
+ }
+ else if (pair.Head().IsAtom() && pair.Tail().IsCallable({"Bool", "Float", "Double", "Int8", "Uint8", "Int16", "Uint16", "Int32", "Uint32", "Int64", "Uint64", "String", "Utf8"})) {
+ writer.WriteKey(pair.Head().Content());
+ if (const auto& type = pair.Tail().Content(); "Bool" == type)
+ writer.Write(FromString<bool>(pair.Tail().Head().Content()));
+ else if ("Float" == type)
+ writer.Write(FromString<float>(pair.Tail().Head().Content()));
+ else if ("Double" == type)
+ writer.Write(FromString<double>(pair.Tail().Head().Content()));
+ else if ("Int8" == type)
+ writer.Write(FromString<i8>(pair.Tail().Head().Content()));
+ else if ("Uint8" == type)
+ writer.Write(FromString<ui8>(pair.Tail().Head().Content()));
+ else if ("Int16" == type)
+ writer.Write(FromString<i16>(pair.Tail().Head().Content()));
+ else if ("Uint16" == type)
+ writer.Write(FromString<ui16>(pair.Tail().Head().Content()));
+ else if ("Int32" == type)
+ writer.Write(FromString<i32>(pair.Tail().Head().Content()));
+ else if ("Uint32" == type)
+ writer.Write(FromString<ui32>(pair.Tail().Head().Content()));
+ else if ("Int64" == type)
+ writer.Write(FromString<i64>(pair.Tail().Head().Content()));
+ else if ("Uint64" == type)
+ writer.Write(FromString<ui64>(pair.Tail().Head().Content()));
+ else
+ writer.Write(pair.Tail().Head().Content());
+
+ }
+ });
+ writer.CloseMap();
+ writer.Flush();
+ if (json == "{}")
+ json.clear();
+ return {std::move(json), std::move(compression)};
+ }
+ }
+ }
+ return {TString(), TString()};
+}
+}
+
+TRuntimeNode BuildParseCall(
+ TRuntimeNode input,
+ const std::string_view& format,
+ const std::string_view& compression,
+ TType* inputItemType,
+ TType* outputItemType,
+ NCommon::TMkqlBuildContext& ctx)
+{
+ if (!compression.empty()) {
+ input = ctx.ProgramBuilder.Map(input, [&ctx, &compression](TRuntimeNode item) {
+ return ctx.ProgramBuilder.Apply(ctx.ProgramBuilder.Udf(std::string("Decompress.") += compression), {item});
+ });
+ }
+
+ const auto structType = static_cast<const TStructType*>(outputItemType);
+ if (format == "raw") {
+ MKQL_ENSURE(1U == structType->GetMembersCount(), "Expected single column.");
+ bool isOptional;
+ const auto schemeType = UnpackOptionalData(structType->GetMemberType(0U), isOptional)->GetSchemeType();
return ctx.ProgramBuilder.ExpandMap(ctx.ProgramBuilder.ToFlow(input),
[&](TRuntimeNode item)->TRuntimeNode::TList {
return { NUdf::TDataType<const char*>::Id == schemeType ?
@@ -120,43 +120,43 @@ TRuntimeNode BuildParseCall(
} else {
const auto userType = ctx.ProgramBuilder.NewTupleType({ctx.ProgramBuilder.NewTupleType({inputItemType}), ctx.ProgramBuilder.NewStructType({}), outputItemType});
input = ctx.ProgramBuilder.ToFlow(ctx.ProgramBuilder.Apply(ctx.ProgramBuilder.Udf("ClickHouseClient.ParseFormat", {}, userType, format), {input}));
- }
-
+ }
+
return ctx.ProgramBuilder.ExpandMap(input,
- [&](TRuntimeNode item) {
- TRuntimeNode::TList fields;
- fields.reserve(structType->GetMembersCount());
- auto j = 0U;
- std::generate_n(std::back_inserter(fields), structType->GetMembersCount(), [&](){ return ctx.ProgramBuilder.Member(item, structType->GetMemberName(j++)); });
- return fields;
- });
-}
-
-NCommon::IMkqlCallableCompiler::TCompiler BuildCompiler(const std::string_view& providerName) {
- return [providerName](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) {
- if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == providerName) {
- const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx);
- const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *wrapper.Input().Ref().GetTypeAnn(), ctx.ProgramBuilder);
- const auto outputItemType = NCommon::BuildType(wrapper.RowType().Ref(), *wrapper.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder);
- const auto& settings = GetSettings(wrapper.Settings().Cast().Ref());
- return BuildParseCall(input, GetFormat(wrapper.Settings().Cast().Ref()).Content() + settings.front(), settings.back(), inputItemType, outputItemType, ctx);
- }
-
- return TRuntimeNode();
- };
-}
-
-TMaybe<TRuntimeNode> TryWrapWithParser(const TDqSourceWideWrap& wrapper, NCommon::TMkqlBuildContext& ctx) {
- const auto& format = GetFormat(wrapper.Settings().Cast().Ref());
- if (!format.Content()) {
- return TMaybe<TRuntimeNode>();
- }
-
- const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx);
- const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *wrapper.Input().Ref().GetTypeAnn(), ctx.ProgramBuilder);
- const auto outputItemType = NCommon::BuildType(wrapper.RowType().Ref(), *wrapper.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder);
- const auto& settings = GetSettings(wrapper.Settings().Cast().Ref());
- return BuildParseCall(input, format.Content() + settings.front(), settings.back(), inputItemType, outputItemType, ctx);
-}
-
-}
+ [&](TRuntimeNode item) {
+ TRuntimeNode::TList fields;
+ fields.reserve(structType->GetMembersCount());
+ auto j = 0U;
+ std::generate_n(std::back_inserter(fields), structType->GetMembersCount(), [&](){ return ctx.ProgramBuilder.Member(item, structType->GetMemberName(j++)); });
+ return fields;
+ });
+}
+
+NCommon::IMkqlCallableCompiler::TCompiler BuildCompiler(const std::string_view& providerName) {
+ return [providerName](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) {
+ if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == providerName) {
+ const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx);
+ const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *wrapper.Input().Ref().GetTypeAnn(), ctx.ProgramBuilder);
+ const auto outputItemType = NCommon::BuildType(wrapper.RowType().Ref(), *wrapper.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder);
+ const auto& settings = GetSettings(wrapper.Settings().Cast().Ref());
+ return BuildParseCall(input, GetFormat(wrapper.Settings().Cast().Ref()).Content() + settings.front(), settings.back(), inputItemType, outputItemType, ctx);
+ }
+
+ return TRuntimeNode();
+ };
+}
+
+TMaybe<TRuntimeNode> TryWrapWithParser(const TDqSourceWideWrap& wrapper, NCommon::TMkqlBuildContext& ctx) {
+ const auto& format = GetFormat(wrapper.Settings().Cast().Ref());
+ if (!format.Content()) {
+ return TMaybe<TRuntimeNode>();
+ }
+
+ const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx);
+ const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *wrapper.Input().Ref().GetTypeAnn(), ctx.ProgramBuilder);
+ const auto outputItemType = NCommon::BuildType(wrapper.RowType().Ref(), *wrapper.RowType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType(), ctx.ProgramBuilder);
+ const auto& settings = GetSettings(wrapper.Settings().Cast().Ref());
+ return BuildParseCall(input, format.Content() + settings.front(), settings.back(), inputItemType, outputItemType, ctx);
+}
+
+}
diff --git a/ydb/library/yql/providers/common/mkql/parser.h b/ydb/library/yql/providers/common/mkql/parser.h
index 1a6e821c179..285366a3867 100644
--- a/ydb/library/yql/providers/common/mkql/parser.h
+++ b/ydb/library/yql/providers/common/mkql/parser.h
@@ -1,26 +1,26 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/ast/yql_expr.h>
-
+
#include <ydb/library/yql/core/yql_opt_utils.h>
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h>
#include <ydb/library/yql/providers/common/mkql/yql_provider_mkql.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_node.h>
-
-#include <algorithm>
-
-namespace NYql {
-
-NKikimr::NMiniKQL::TRuntimeNode BuildParseCall(
- NKikimr::NMiniKQL::TRuntimeNode input,
- const std::string_view& format,
- const std::string_view& compression,
- NKikimr::NMiniKQL::TType* inputItemType,
- NKikimr::NMiniKQL::TType* outputItemType,
- NCommon::TMkqlBuildContext& ctx);
-
-TMaybe<NKikimr::NMiniKQL::TRuntimeNode> TryWrapWithParser(const NYql::NNodes::TDqSourceWideWrap& wrapper, NCommon::TMkqlBuildContext& ctx);
-
-}
+
+#include <algorithm>
+
+namespace NYql {
+
+NKikimr::NMiniKQL::TRuntimeNode BuildParseCall(
+ NKikimr::NMiniKQL::TRuntimeNode input,
+ const std::string_view& format,
+ const std::string_view& compression,
+ NKikimr::NMiniKQL::TType* inputItemType,
+ NKikimr::NMiniKQL::TType* outputItemType,
+ NCommon::TMkqlBuildContext& ctx);
+
+TMaybe<NKikimr::NMiniKQL::TRuntimeNode> TryWrapWithParser(const NYql::NNodes::TDqSourceWideWrap& wrapper, NCommon::TMkqlBuildContext& ctx);
+
+}
diff --git a/ydb/library/yql/providers/common/mkql/ya.make b/ydb/library/yql/providers/common/mkql/ya.make
index 1c53310c007..8de943fad2e 100644
--- a/ydb/library/yql/providers/common/mkql/ya.make
+++ b/ydb/library/yql/providers/common/mkql/ya.make
@@ -7,8 +7,8 @@ OWNER(
)
SRCS(
- parser.cpp
- parser.h
+ parser.cpp
+ parser.h
yql_provider_mkql.cpp
yql_provider_mkql.h
yql_type_mkql.cpp
@@ -16,7 +16,7 @@ SRCS(
)
PEERDIR(
- library/cpp/json
+ library/cpp/json
ydb/library/yql/ast
ydb/library/yql/minikql
ydb/library/yql/public/decimal
@@ -30,5 +30,5 @@ PEERDIR(
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
index bde9e215b62..dd1f162d784 100644
--- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
+++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
@@ -1569,75 +1569,75 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
AddCallable("HoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) {
const auto stream = MkqlBuildExpr(node.Head(), ctx);
- const auto timeExtractor = [&](TRuntimeNode item) {
+ const auto timeExtractor = [&](TRuntimeNode item) {
return MkqlBuildLambda(*node.Child(1), ctx, {item});
};
- const auto hop = MkqlBuildExpr(*node.Child(2), ctx);
- const auto interval = MkqlBuildExpr(*node.Child(3), ctx);
- const auto delay = MkqlBuildExpr(*node.Child(4), ctx);
-
- const auto init = [&](TRuntimeNode item) {
- return MkqlBuildLambda(*node.Child(5), ctx, {item});
- };
+ const auto hop = MkqlBuildExpr(*node.Child(2), ctx);
+ const auto interval = MkqlBuildExpr(*node.Child(3), ctx);
+ const auto delay = MkqlBuildExpr(*node.Child(4), ctx);
+
+ const auto init = [&](TRuntimeNode item) {
+ return MkqlBuildLambda(*node.Child(5), ctx, {item});
+ };
const auto update = [&](TRuntimeNode item, TRuntimeNode state) {
- return MkqlBuildLambda(*node.Child(6), ctx, {item, state});
+ return MkqlBuildLambda(*node.Child(6), ctx, {item, state});
};
const auto save = node.Child(3)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) {
- return MkqlBuildLambda(*node.Child(7), ctx, {state});
+ return MkqlBuildLambda(*node.Child(7), ctx, {state});
};
const auto load = node.Child(4)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) {
- return MkqlBuildLambda(*node.Child(8), ctx, {state});
+ return MkqlBuildLambda(*node.Child(8), ctx, {state});
};
const auto merge = [&](TRuntimeNode state1, TRuntimeNode state2) {
- return MkqlBuildLambda(*node.Child(9), ctx, {state1, state2});
+ return MkqlBuildLambda(*node.Child(9), ctx, {state1, state2});
};
const auto finish = [&](TRuntimeNode state, TRuntimeNode time) {
- return MkqlBuildLambda(*node.Child(10), ctx, {state, time});
+ return MkqlBuildLambda(*node.Child(10), ctx, {state, time});
};
return ctx.ProgramBuilder.HoppingCore(
stream, timeExtractor, init, update, save, load, merge, finish, hop, interval, delay);
});
- AddCallable("MultiHoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) {
- const auto stream = MkqlBuildExpr(node.Head(), ctx);
-
- const auto keyExtractor = [&](TRuntimeNode item) {
- return MkqlBuildLambda(*node.Child(1), ctx, {item});
- };
-
- const auto timeExtractor = [&](TRuntimeNode item) {
- return MkqlBuildLambda(*node.Child(2), ctx, {item});
- };
- const auto hop = MkqlBuildExpr(*node.Child(3), ctx);
- const auto interval = MkqlBuildExpr(*node.Child(4), ctx);
- const auto delay = MkqlBuildExpr(*node.Child(5), ctx);
- const auto dataWatermarks = ctx.ProgramBuilder.NewDataLiteral(FromString<bool>(*node.Child(6), NUdf::EDataSlot::Bool));
-
- const auto init = [&](TRuntimeNode item) {
- return MkqlBuildLambda(*node.Child(7), ctx, {item});
- };
- const auto update = [&](TRuntimeNode item, TRuntimeNode state) {
- return MkqlBuildLambda(*node.Child(8), ctx, {item, state});
- };
- const auto save = node.Child(3)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) {
- return MkqlBuildLambda(*node.Child(9), ctx, {state});
- };
- const auto load = node.Child(4)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) {
- return MkqlBuildLambda(*node.Child(10), ctx, {state});
- };
- const auto merge = [&](TRuntimeNode state1, TRuntimeNode state2) {
- return MkqlBuildLambda(*node.Child(11), ctx, {state1, state2});
- };
- const auto finish = [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) {
- return MkqlBuildLambda(*node.Child(12), ctx, {key, state, time});
- };
-
- return ctx.ProgramBuilder.MultiHoppingCore(
- stream, keyExtractor, timeExtractor, init, update, save, load, merge, finish,
- hop, interval, delay, dataWatermarks);
- });
-
+ AddCallable("MultiHoppingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) {
+ const auto stream = MkqlBuildExpr(node.Head(), ctx);
+
+ const auto keyExtractor = [&](TRuntimeNode item) {
+ return MkqlBuildLambda(*node.Child(1), ctx, {item});
+ };
+
+ const auto timeExtractor = [&](TRuntimeNode item) {
+ return MkqlBuildLambda(*node.Child(2), ctx, {item});
+ };
+ const auto hop = MkqlBuildExpr(*node.Child(3), ctx);
+ const auto interval = MkqlBuildExpr(*node.Child(4), ctx);
+ const auto delay = MkqlBuildExpr(*node.Child(5), ctx);
+ const auto dataWatermarks = ctx.ProgramBuilder.NewDataLiteral(FromString<bool>(*node.Child(6), NUdf::EDataSlot::Bool));
+
+ const auto init = [&](TRuntimeNode item) {
+ return MkqlBuildLambda(*node.Child(7), ctx, {item});
+ };
+ const auto update = [&](TRuntimeNode item, TRuntimeNode state) {
+ return MkqlBuildLambda(*node.Child(8), ctx, {item, state});
+ };
+ const auto save = node.Child(3)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) {
+ return MkqlBuildLambda(*node.Child(9), ctx, {state});
+ };
+ const auto load = node.Child(4)->IsCallable("Void") ? std::function<TRuntimeNode(TRuntimeNode)>() : [&](TRuntimeNode state) {
+ return MkqlBuildLambda(*node.Child(10), ctx, {state});
+ };
+ const auto merge = [&](TRuntimeNode state1, TRuntimeNode state2) {
+ return MkqlBuildLambda(*node.Child(11), ctx, {state1, state2});
+ };
+ const auto finish = [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) {
+ return MkqlBuildLambda(*node.Child(12), ctx, {key, state, time});
+ };
+
+ return ctx.ProgramBuilder.MultiHoppingCore(
+ stream, keyExtractor, timeExtractor, init, update, save, load, merge, finish,
+ hop, interval, delay, dataWatermarks);
+ });
+
AddCallable("ToDict", [](const TExprNode& node, TMkqlBuildContext& ctx) {
const auto list = MkqlBuildExpr(node.Head(), ctx);
TMaybe<bool> isMany;
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 4f36ed8a710..8561fe25368 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -334,25 +334,25 @@ message TS3GatewayConfig {
///////////////////////////// Solomon /////////////////////////////
message TSolomonClusterConfig {
- enum ESolomonClusterType {
- SCT_UNSPECIFIED = 0;
- SCT_SOLOMON = 1;
- SCT_MONITORING = 2;
- }
-
- message TShardPath {
- required string Project = 1; // CloudId for YC
- required string Cluster = 2; // FolderId for YC
- }
-
+ enum ESolomonClusterType {
+ SCT_UNSPECIFIED = 0;
+ SCT_SOLOMON = 1;
+ SCT_MONITORING = 2;
+ }
+
+ message TShardPath {
+ required string Project = 1; // CloudId for YC
+ required string Cluster = 2; // FolderId for YC
+ }
+
optional string Name = 1;
optional string Cluster = 2;
optional bool UseSsl = 3;
- optional ESolomonClusterType ClusterType = 4 [default = SCT_SOLOMON];
+ optional ESolomonClusterType ClusterType = 4 [default = SCT_SOLOMON];
optional string Token = 5;
optional string ServiceAccountId = 6;
optional string ServiceAccountIdSignature = 7;
- optional TShardPath Path = 8;
+ optional TShardPath Path = 8;
repeated TAttr Settings = 100;
}
diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp
index 96c5076fd0f..aa782ee93ff 100644
--- a/ydb/library/yql/providers/common/provider/yql_provider.cpp
+++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp
@@ -574,16 +574,16 @@ static void GetToken(const TString& string, TString& out, const TTypeAnnotationC
}
}
}
- for (auto& x : type.DataSinks) {
- auto tokens = x->GetClusterTokens();
- if (tokens) {
- auto token = tokens->FindPtr(clusterName);
- if (token) {
- out = *token;
- return;
- }
- }
- }
+ for (auto& x : type.DataSinks) {
+ auto tokens = x->GetClusterTokens();
+ if (tokens) {
+ auto token = tokens->FindPtr(clusterName);
+ if (token) {
+ out = *token;
+ return;
+ }
+ }
+ }
}
}
diff --git a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp
index 97592bac3e0..39cae924364 100644
--- a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp
+++ b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.cpp
@@ -1,143 +1,143 @@
-#include "dq_fake_ca.h"
-
+#include "dq_fake_ca.h"
+
#include <ydb/library/yql/minikql/mkql_string_util.h>
-
+
#include <ydb/core/testlib/basics/appdata.h>
-
-#include <util/system/env.h>
-
-#include <condition_variable>
-#include <thread>
-
-namespace NYql::NDq {
-
-using namespace NActors;
-
-NYql::NDqProto::TCheckpoint CreateCheckpoint(ui64 id) {
- auto checkpoint = NYql::NDqProto::TCheckpoint();
- checkpoint.SetGeneration(0);
- checkpoint.SetId(id);
- return checkpoint;
-}
-
-TFakeActor::TFakeActor(TSourcePromises& sourcePromises, TSinkPromises& sinkPromises)
- : TActor<TFakeActor>(&TFakeActor::StateFunc)
- , MemoryInfo("test")
- , HolderFactory(Alloc.Ref(), MemoryInfo)
- , SourceCallbacks(*this)
- , SinkCallbacks(*this)
- , SourcePromises(sourcePromises)
- , SinkPromises(sinkPromises)
-{
- Alloc.Release();
-}
-
-TFakeActor::~TFakeActor() {
- Alloc.Acquire();
-}
-
-void TFakeActor::InitSink(IDqSinkActor* dqSink, IActor* dqSinkAsActor) {
- DqSinkActorId = RegisterWithSameMailbox(dqSinkAsActor),
- DqSinkActor = dqSink;
- DqSinkActorAsActor = dqSinkAsActor;
-}
-
-void TFakeActor::InitSource(IDqSourceActor* dqSource, IActor* dqSourceAsActor) {
- DqSourceActorId = RegisterWithSameMailbox(dqSourceAsActor),
- DqSourceActor = dqSource;
- DqSourceActorAsActor = dqSourceAsActor;
-}
-
-void TFakeActor::Terminate() {
- if (DqSourceActorId) {
+
+#include <util/system/env.h>
+
+#include <condition_variable>
+#include <thread>
+
+namespace NYql::NDq {
+
+using namespace NActors;
+
+NYql::NDqProto::TCheckpoint CreateCheckpoint(ui64 id) {
+ auto checkpoint = NYql::NDqProto::TCheckpoint();
+ checkpoint.SetGeneration(0);
+ checkpoint.SetId(id);
+ return checkpoint;
+}
+
+TFakeActor::TFakeActor(TSourcePromises& sourcePromises, TSinkPromises& sinkPromises)
+ : TActor<TFakeActor>(&TFakeActor::StateFunc)
+ , MemoryInfo("test")
+ , HolderFactory(Alloc.Ref(), MemoryInfo)
+ , SourceCallbacks(*this)
+ , SinkCallbacks(*this)
+ , SourcePromises(sourcePromises)
+ , SinkPromises(sinkPromises)
+{
+ Alloc.Release();
+}
+
+TFakeActor::~TFakeActor() {
+ Alloc.Acquire();
+}
+
+void TFakeActor::InitSink(IDqSinkActor* dqSink, IActor* dqSinkAsActor) {
+ DqSinkActorId = RegisterWithSameMailbox(dqSinkAsActor),
+ DqSinkActor = dqSink;
+ DqSinkActorAsActor = dqSinkAsActor;
+}
+
+void TFakeActor::InitSource(IDqSourceActor* dqSource, IActor* dqSourceAsActor) {
+ DqSourceActorId = RegisterWithSameMailbox(dqSourceAsActor),
+ DqSourceActor = dqSource;
+ DqSourceActorAsActor = dqSourceAsActor;
+}
+
+void TFakeActor::Terminate() {
+ if (DqSourceActorId) {
DqSourceActor->PassAway();
-
- DqSourceActorId = std::nullopt;
- DqSourceActor = nullptr;
- DqSourceActorAsActor = nullptr;
- }
-
- if (DqSinkActorId) {
+
+ DqSourceActorId = std::nullopt;
+ DqSourceActor = nullptr;
+ DqSourceActorAsActor = nullptr;
+ }
+
+ if (DqSinkActorId) {
DqSinkActor->PassAway();
-
- DqSinkActorId = std::nullopt;
- DqSinkActor = nullptr;
- DqSinkActorAsActor = nullptr;
- }
-}
-
-TFakeActor::TSourceCallbacks& TFakeActor::GetSourceCallbacks() {
- return SourceCallbacks;
-}
-
-TFakeActor::TSinkCallbacks& TFakeActor::GetSinkCallbacks() {
- return SinkCallbacks;
-}
-
-NKikimr::NMiniKQL::THolderFactory& TFakeActor::GetHolderFactory() {
- return HolderFactory;
-}
-
-TFakeCASetup::TFakeCASetup()
- : Runtime(new NActors::TTestBasicRuntime(1, true))
- , FakeActorId(0, "FakeActor")
-{
- Runtime->AddLocalService(
- FakeActorId,
- NActors::TActorSetupCmd(
- new TFakeActor(SourcePromises, SinkPromises),
- NActors::TMailboxType::Simple,
- 0));
-
- TAutoPtr<NKikimr::TAppPrepare> app = new NKikimr::TAppPrepare();
- Runtime->Initialize(app->Unwrap());
-}
-
-TFakeCASetup::~TFakeCASetup() {
- Execute([](TFakeActor& actor) {
- actor.Terminate();
- });
-}
-
-void TFakeCASetup::SinkWrite(const TWriteValueProducer valueProducer, TMaybe<NDqProto::TCheckpoint> checkpoint) {
- Execute([&valueProducer, checkpoint](TFakeActor& actor) {
- auto batch = valueProducer(actor.GetHolderFactory());
- Y_ASSERT(actor.DqSinkActor);
- actor.DqSinkActor->SendData(std::move(batch), 0, checkpoint, false);
- });
-}
-
+
+ DqSinkActorId = std::nullopt;
+ DqSinkActor = nullptr;
+ DqSinkActorAsActor = nullptr;
+ }
+}
+
+TFakeActor::TSourceCallbacks& TFakeActor::GetSourceCallbacks() {
+ return SourceCallbacks;
+}
+
+TFakeActor::TSinkCallbacks& TFakeActor::GetSinkCallbacks() {
+ return SinkCallbacks;
+}
+
+NKikimr::NMiniKQL::THolderFactory& TFakeActor::GetHolderFactory() {
+ return HolderFactory;
+}
+
+TFakeCASetup::TFakeCASetup()
+ : Runtime(new NActors::TTestBasicRuntime(1, true))
+ , FakeActorId(0, "FakeActor")
+{
+ Runtime->AddLocalService(
+ FakeActorId,
+ NActors::TActorSetupCmd(
+ new TFakeActor(SourcePromises, SinkPromises),
+ NActors::TMailboxType::Simple,
+ 0));
+
+ TAutoPtr<NKikimr::TAppPrepare> app = new NKikimr::TAppPrepare();
+ Runtime->Initialize(app->Unwrap());
+}
+
+TFakeCASetup::~TFakeCASetup() {
+ Execute([](TFakeActor& actor) {
+ actor.Terminate();
+ });
+}
+
+void TFakeCASetup::SinkWrite(const TWriteValueProducer valueProducer, TMaybe<NDqProto::TCheckpoint> checkpoint) {
+ Execute([&valueProducer, checkpoint](TFakeActor& actor) {
+ auto batch = valueProducer(actor.GetHolderFactory());
+ Y_ASSERT(actor.DqSinkActor);
+ actor.DqSinkActor->SendData(std::move(batch), 0, checkpoint, false);
+ });
+}
+
void TFakeCASetup::SaveSourceState(NDqProto::TCheckpoint checkpoint, NDqProto::TSourceState& state) {
- Execute([&state, &checkpoint](TFakeActor& actor) {
- Y_ASSERT(actor.DqSourceActor);
+ Execute([&state, &checkpoint](TFakeActor& actor) {
+ Y_ASSERT(actor.DqSourceActor);
actor.DqSourceActor->SaveState(checkpoint, state);
- });
-}
-
+ });
+}
+
void TFakeCASetup::LoadSource(const NDqProto::TSourceState& state) {
- Execute([&state](TFakeActor& actor) {
- Y_ASSERT(actor.DqSourceActor);
- actor.DqSourceActor->LoadState(state);
- });
-}
-
+ Execute([&state](TFakeActor& actor) {
+ Y_ASSERT(actor.DqSourceActor);
+ actor.DqSourceActor->LoadState(state);
+ });
+}
+
void TFakeCASetup::LoadSink(const NDqProto::TSinkState& state) {
- Execute([&state](TFakeActor& actor) {
- Y_ASSERT(actor.DqSinkActor);
- actor.DqSinkActor->LoadState(state);
- });
-}
-
-void TFakeCASetup::Execute(TCallback callback) {
- std::exception_ptr exception_ptr = nullptr;
- const TActorId& edgeId = Runtime->AllocateEdgeActor();
- auto promise = NThreading::NewPromise();
- Runtime->Send(new IEventHandle(FakeActorId, edgeId, new TEvPrivate::TEvExecute(promise, callback, exception_ptr)));
- auto future = promise.GetFuture();
- future.Wait();
- if (exception_ptr) {
- std::rethrow_exception(exception_ptr);
- }
-}
-
-} // namespace NKikimr::NMiniKQL
+ Execute([&state](TFakeActor& actor) {
+ Y_ASSERT(actor.DqSinkActor);
+ actor.DqSinkActor->LoadState(state);
+ });
+}
+
+void TFakeCASetup::Execute(TCallback callback) {
+ std::exception_ptr exception_ptr = nullptr;
+ const TActorId& edgeId = Runtime->AllocateEdgeActor();
+ auto promise = NThreading::NewPromise();
+ Runtime->Send(new IEventHandle(FakeActorId, edgeId, new TEvPrivate::TEvExecute(promise, callback, exception_ptr)));
+ auto future = promise.GetFuture();
+ future.Wait();
+ if (exception_ptr) {
+ std::rethrow_exception(exception_ptr);
+ }
+}
+
+} // namespace NKikimr::NMiniKQL
diff --git a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h
index 96fca4f4650..94d815a1eb8 100644
--- a/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h
+++ b/ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h
@@ -1,223 +1,223 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h>
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h>
#include <ydb/library/yql/minikql/mkql_alloc.h>
-
+
#include <ydb/core/testlib/basics/runtime.h>
-
-#include <library/cpp/retry/retry.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <chrono>
-#include <queue>
-
-namespace NYql::NDq {
-
-class TFakeActor;
-
-using TRuntimePtr = std::unique_ptr<NActors::TTestActorRuntime>;
-using TCallback = std::function<void(TFakeActor&)>;
-template<typename T>
-using TReadValueParser = std::function<std::vector<T>(const NUdf::TUnboxedValue&)>;
-using TWriteValueProducer = std::function<NKikimr::NMiniKQL::TUnboxedValueVector(NKikimr::NMiniKQL::THolderFactory&)>;
-
-namespace {
- struct TEvPrivate {
- // Event ids
- enum EEv : ui32 {
- EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE),
-
- EvExecute = EvBegin,
-
- EvEnd
- };
-
- static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)");
-
- // Events
-
- struct TEvExecute : public NActors::TEventLocal<TEvExecute, EvExecute> {
- TEvExecute(NThreading::TPromise<void>& promise, TCallback callback, std::exception_ptr& resultException)
- : Promise(promise)
- , Callback(callback)
- , ResultException(resultException)
- {}
-
- NThreading::TPromise<void> Promise;
- TCallback Callback;
- std::exception_ptr& ResultException;
- };
- };
-}
-
-struct TSourcePromises {
- NThreading::TPromise<void> NewSourceDataArrived = NThreading::NewPromise();
- NThreading::TPromise<TIssues> FatalError = NThreading::NewPromise<TIssues>();
-};
-
-struct TSinkPromises {
- NThreading::TPromise<void> ResumeExecution = NThreading::NewPromise();
- NThreading::TPromise<TIssues> Issue = NThreading::NewPromise<TIssues>();
+
+#include <library/cpp/retry/retry.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <chrono>
+#include <queue>
+
+namespace NYql::NDq {
+
+class TFakeActor;
+
+using TRuntimePtr = std::unique_ptr<NActors::TTestActorRuntime>;
+using TCallback = std::function<void(TFakeActor&)>;
+template<typename T>
+using TReadValueParser = std::function<std::vector<T>(const NUdf::TUnboxedValue&)>;
+using TWriteValueProducer = std::function<NKikimr::NMiniKQL::TUnboxedValueVector(NKikimr::NMiniKQL::THolderFactory&)>;
+
+namespace {
+ struct TEvPrivate {
+ // Event ids
+ enum EEv : ui32 {
+ EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE),
+
+ EvExecute = EvBegin,
+
+ EvEnd
+ };
+
+ static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)");
+
+ // Events
+
+ struct TEvExecute : public NActors::TEventLocal<TEvExecute, EvExecute> {
+ TEvExecute(NThreading::TPromise<void>& promise, TCallback callback, std::exception_ptr& resultException)
+ : Promise(promise)
+ , Callback(callback)
+ , ResultException(resultException)
+ {}
+
+ NThreading::TPromise<void> Promise;
+ TCallback Callback;
+ std::exception_ptr& ResultException;
+ };
+ };
+}
+
+struct TSourcePromises {
+ NThreading::TPromise<void> NewSourceDataArrived = NThreading::NewPromise();
+ NThreading::TPromise<TIssues> FatalError = NThreading::NewPromise<TIssues>();
+};
+
+struct TSinkPromises {
+ NThreading::TPromise<void> ResumeExecution = NThreading::NewPromise();
+ NThreading::TPromise<TIssues> Issue = NThreading::NewPromise<TIssues>();
NThreading::TPromise<NDqProto::TSinkState> StateSaved = NThreading::NewPromise<NDqProto::TSinkState>();
-};
-
-NYql::NDqProto::TCheckpoint CreateCheckpoint(ui64 id = 0);
-
-class TFakeActor : public NActors::TActor<TFakeActor> {
- struct TSourceCallbacks : public IDqSourceActor::ICallbacks {
- explicit TSourceCallbacks(TFakeActor& parent) : Parent(parent) {}
-
- void OnNewSourceDataArrived(ui64) override {
- Parent.SourcePromises.NewSourceDataArrived.SetValue();
- Parent.SourcePromises.NewSourceDataArrived = NThreading::NewPromise();
- }
-
- void OnSourceError(ui64, const TIssues& issues, bool isFatal) override {
- Y_UNUSED(isFatal);
- Parent.SourcePromises.FatalError.SetValue(issues);
- Parent.SourcePromises.FatalError = NThreading::NewPromise<TIssues>();
- }
-
- TFakeActor& Parent;
- };
-
- struct TSinkCallbacks : public IDqSinkActor::ICallbacks {
- explicit TSinkCallbacks(TFakeActor& parent) : Parent(parent) {}
-
- void ResumeExecution() override {
- Parent.SinkPromises.ResumeExecution.SetValue();
- Parent.SinkPromises.ResumeExecution = NThreading::NewPromise();
- };
-
- void OnSinkError(ui64, const TIssues& issues, bool isFatal) override {
- Y_UNUSED(isFatal);
- Parent.SinkPromises.Issue.SetValue(issues);
- Parent.SinkPromises.Issue = NThreading::NewPromise<TIssues>();
- };
-
+};
+
+NYql::NDqProto::TCheckpoint CreateCheckpoint(ui64 id = 0);
+
+class TFakeActor : public NActors::TActor<TFakeActor> {
+ struct TSourceCallbacks : public IDqSourceActor::ICallbacks {
+ explicit TSourceCallbacks(TFakeActor& parent) : Parent(parent) {}
+
+ void OnNewSourceDataArrived(ui64) override {
+ Parent.SourcePromises.NewSourceDataArrived.SetValue();
+ Parent.SourcePromises.NewSourceDataArrived = NThreading::NewPromise();
+ }
+
+ void OnSourceError(ui64, const TIssues& issues, bool isFatal) override {
+ Y_UNUSED(isFatal);
+ Parent.SourcePromises.FatalError.SetValue(issues);
+ Parent.SourcePromises.FatalError = NThreading::NewPromise<TIssues>();
+ }
+
+ TFakeActor& Parent;
+ };
+
+ struct TSinkCallbacks : public IDqSinkActor::ICallbacks {
+ explicit TSinkCallbacks(TFakeActor& parent) : Parent(parent) {}
+
+ void ResumeExecution() override {
+ Parent.SinkPromises.ResumeExecution.SetValue();
+ Parent.SinkPromises.ResumeExecution = NThreading::NewPromise();
+ };
+
+ void OnSinkError(ui64, const TIssues& issues, bool isFatal) override {
+ Y_UNUSED(isFatal);
+ Parent.SinkPromises.Issue.SetValue(issues);
+ Parent.SinkPromises.Issue = NThreading::NewPromise<TIssues>();
+ };
+
void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint&) override {
- Y_UNUSED(outputIndex);
- Parent.SinkPromises.StateSaved.SetValue(state);
+ Y_UNUSED(outputIndex);
+ Parent.SinkPromises.StateSaved.SetValue(state);
Parent.SinkPromises.StateSaved = NThreading::NewPromise<NDqProto::TSinkState>();
- };
-
- TFakeActor& Parent;
- };
-
-public:
- TFakeActor(TSourcePromises& sourcePromises, TSinkPromises& sinkPromises);
- ~TFakeActor();
-
- void InitSink(IDqSinkActor* dqSink, IActor* dqSinkAsActor);
- void InitSource(IDqSourceActor* dqSource, IActor* dqSourceAsActor);
- void Terminate();
-
- TSourceCallbacks& GetSourceCallbacks();
- TSinkCallbacks& GetSinkCallbacks();
- NKikimr::NMiniKQL::THolderFactory& GetHolderFactory();
-
-public:
- IDqSourceActor* DqSourceActor = nullptr;
- IDqSinkActor* DqSinkActor = nullptr;
-
-private:
- STRICT_STFUNC(StateFunc,
- hFunc(TEvPrivate::TEvExecute, Handle);
- )
-
- void Handle(TEvPrivate::TEvExecute::TPtr& ev) {
- TGuard<NKikimr::NMiniKQL::TScopedAlloc> guard(Alloc);
- try {
- ev->Get()->Callback(*this);
- } catch (...) {
- ev->Get()->ResultException = std::current_exception();
- }
- ev->Get()->Promise.SetValue();
- }
-
-private:
- NKikimr::NMiniKQL::TScopedAlloc Alloc;
- NKikimr::NMiniKQL::TMemoryUsageInfo MemoryInfo;
- NKikimr::NMiniKQL::THolderFactory HolderFactory;
-
- std::optional<NActors::TActorId> DqSourceActorId;
- IActor* DqSourceActorAsActor = nullptr;
-
- std::optional<NActors::TActorId> DqSinkActorId;
- IActor* DqSinkActorAsActor = nullptr;
-
- TSourceCallbacks SourceCallbacks;
- TSinkCallbacks SinkCallbacks;
-
- TSourcePromises& SourcePromises;
- TSinkPromises& SinkPromises;
-};
-
-struct TFakeCASetup {
- TFakeCASetup();
- ~TFakeCASetup();
-
- template<typename T>
- std::vector<T> SourceRead(const TReadValueParser<T> parser, i64 freeSpace = 12345) {
- std::vector<T> result;
- Execute([&result, &parser, freeSpace](TFakeActor& actor) {
- NKikimr::NMiniKQL::TUnboxedValueVector buffer;
- bool finished = false;
- actor.DqSourceActor->GetSourceData(buffer, finished, freeSpace);
-
- for (const auto& uv : buffer) {
- for (const auto item : parser(uv)) {
- result.emplace_back(item);
- }
- }
- });
-
- return result;
- }
-
- template<typename T>
- std::vector<T> SourceReadUntil(
- const TReadValueParser<T> parser,
- ui64 size,
- i64 eachReadFreeSpace = 1000,
- TDuration timeout = TDuration::Seconds(10))
- {
- std::vector<T> result;
- DoWithRetry([&](){
- auto batch = SourceRead<T>(parser, eachReadFreeSpace);
- for (const auto& item : batch) {
- result.emplace_back(item);
- }
-
- if (result.size() < size) {
- SourcePromises.NewSourceDataArrived.GetFuture().Wait(timeout);
- ythrow yexception() << "Not enough data";
- }
- },
- TRetryOptions(3),
- false);
-
- return result;
- }
-
- void SinkWrite(const TWriteValueProducer valueProducer, TMaybe<NDqProto::TCheckpoint> checkpoint = Nothing());
-
+ };
+
+ TFakeActor& Parent;
+ };
+
+public:
+ TFakeActor(TSourcePromises& sourcePromises, TSinkPromises& sinkPromises);
+ ~TFakeActor();
+
+ void InitSink(IDqSinkActor* dqSink, IActor* dqSinkAsActor);
+ void InitSource(IDqSourceActor* dqSource, IActor* dqSourceAsActor);
+ void Terminate();
+
+ TSourceCallbacks& GetSourceCallbacks();
+ TSinkCallbacks& GetSinkCallbacks();
+ NKikimr::NMiniKQL::THolderFactory& GetHolderFactory();
+
+public:
+ IDqSourceActor* DqSourceActor = nullptr;
+ IDqSinkActor* DqSinkActor = nullptr;
+
+private:
+ STRICT_STFUNC(StateFunc,
+ hFunc(TEvPrivate::TEvExecute, Handle);
+ )
+
+ void Handle(TEvPrivate::TEvExecute::TPtr& ev) {
+ TGuard<NKikimr::NMiniKQL::TScopedAlloc> guard(Alloc);
+ try {
+ ev->Get()->Callback(*this);
+ } catch (...) {
+ ev->Get()->ResultException = std::current_exception();
+ }
+ ev->Get()->Promise.SetValue();
+ }
+
+private:
+ NKikimr::NMiniKQL::TScopedAlloc Alloc;
+ NKikimr::NMiniKQL::TMemoryUsageInfo MemoryInfo;
+ NKikimr::NMiniKQL::THolderFactory HolderFactory;
+
+ std::optional<NActors::TActorId> DqSourceActorId;
+ IActor* DqSourceActorAsActor = nullptr;
+
+ std::optional<NActors::TActorId> DqSinkActorId;
+ IActor* DqSinkActorAsActor = nullptr;
+
+ TSourceCallbacks SourceCallbacks;
+ TSinkCallbacks SinkCallbacks;
+
+ TSourcePromises& SourcePromises;
+ TSinkPromises& SinkPromises;
+};
+
+struct TFakeCASetup {
+ TFakeCASetup();
+ ~TFakeCASetup();
+
+ template<typename T>
+ std::vector<T> SourceRead(const TReadValueParser<T> parser, i64 freeSpace = 12345) {
+ std::vector<T> result;
+ Execute([&result, &parser, freeSpace](TFakeActor& actor) {
+ NKikimr::NMiniKQL::TUnboxedValueVector buffer;
+ bool finished = false;
+ actor.DqSourceActor->GetSourceData(buffer, finished, freeSpace);
+
+ for (const auto& uv : buffer) {
+ for (const auto item : parser(uv)) {
+ result.emplace_back(item);
+ }
+ }
+ });
+
+ return result;
+ }
+
+ template<typename T>
+ std::vector<T> SourceReadUntil(
+ const TReadValueParser<T> parser,
+ ui64 size,
+ i64 eachReadFreeSpace = 1000,
+ TDuration timeout = TDuration::Seconds(10))
+ {
+ std::vector<T> result;
+ DoWithRetry([&](){
+ auto batch = SourceRead<T>(parser, eachReadFreeSpace);
+ for (const auto& item : batch) {
+ result.emplace_back(item);
+ }
+
+ if (result.size() < size) {
+ SourcePromises.NewSourceDataArrived.GetFuture().Wait(timeout);
+ ythrow yexception() << "Not enough data";
+ }
+ },
+ TRetryOptions(3),
+ false);
+
+ return result;
+ }
+
+ void SinkWrite(const TWriteValueProducer valueProducer, TMaybe<NDqProto::TCheckpoint> checkpoint = Nothing());
+
void SaveSourceState(NDqProto::TCheckpoint checkpoint, NDqProto::TSourceState& state);
-
+
void LoadSource(const NDqProto::TSourceState& state);
void LoadSink(const NDqProto::TSinkState& state);
-
- void Execute(TCallback callback);
-
-public:
- TRuntimePtr Runtime;
- NActors::TActorId FakeActorId;
- TSourcePromises SourcePromises;
- TSinkPromises SinkPromises;
-};
-
-} // namespace NKikimr::NMiniKQL
+
+ void Execute(TCallback callback);
+
+public:
+ TRuntimePtr Runtime;
+ NActors::TActorId FakeActorId;
+ TSourcePromises SourcePromises;
+ TSinkPromises SinkPromises;
+};
+
+} // namespace NKikimr::NMiniKQL
diff --git a/ydb/library/yql/providers/common/ut_helpers/ya.make b/ydb/library/yql/providers/common/ut_helpers/ya.make
index cf95837bb36..14cb9b41a6e 100644
--- a/ydb/library/yql/providers/common/ut_helpers/ya.make
+++ b/ydb/library/yql/providers/common/ut_helpers/ya.make
@@ -1,21 +1,21 @@
-LIBRARY()
-
-OWNER(g:yql)
-
-SRCS(
- dq_fake_ca.cpp
-)
-
-PEERDIR(
- library/cpp/retry
+LIBRARY()
+
+OWNER(g:yql)
+
+SRCS(
+ dq_fake_ca.cpp
+)
+
+PEERDIR(
+ library/cpp/retry
ydb/core/testlib/basics
ydb/library/yql/minikql
ydb/library/yql/minikql/computation
ydb/library/yql/public/udf/service/exception_policy
ydb/library/yql/sql
ydb/library/yql/providers/common/comp_nodes
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/providers/dq/actors/task_controller.cpp b/ydb/library/yql/providers/dq/actors/task_controller.cpp
index 17cf0a4c550..238828e453b 100644
--- a/ydb/library/yql/providers/dq/actors/task_controller.cpp
+++ b/ydb/library/yql/providers/dq/actors/task_controller.cpp
@@ -221,8 +221,8 @@ private:
publicCounterName = "query.cpu_usage_us";
isDeriv = true;
} else if (name == "Bytes") {
- if (labels.find("Source") != labels.end()) publicCounterName = "query.input_bytes";
- else if (labels.find("Sink") != labels.end()) publicCounterName = "query.output_bytes";
+ if (labels.find("Source") != labels.end()) publicCounterName = "query.input_bytes";
+ else if (labels.find("Sink") != labels.end()) publicCounterName = "query.output_bytes";
isDeriv = true;
}
if (publicCounterName) {
@@ -273,10 +273,10 @@ private:
TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "MkqlMaxMemoryUsage"), v);
}
- for (const auto& stat : s.GetMkqlStats()) {
- TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, stat.GetName()), stat.GetValue());
- }
-
+ for (const auto& stat : s.GetMkqlStats()) {
+ TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, stat.GetName()), stat.GetValue());
+ }
+
if (stats.ComputeCpuTimeByRunSize()) {
auto& hist = TaskStat.GetHistogram(TaskStat.GetCounterName("TaskRunner", labels, "ComputeTimeByRunMs"));
for (const auto& bucket : s.GetComputeCpuTimeByRun()) {
@@ -340,44 +340,44 @@ private:
// }
}
- for (const auto& stats : s.GetSources()) {
- std::map<TString, TString> labels = {
- {"Task", ToString(taskId)},
+ for (const auto& stats : s.GetSources()) {
+ std::map<TString, TString> labels = {
+ {"Task", ToString(taskId)},
{"Source", ToString(stats.GetInputIndex())}
- };
-
- ADD_COUNTER(Chunks);
- ADD_COUNTER(Bytes);
- ADD_COUNTER(RowsIn);
- ADD_COUNTER(RowsOut);
- ADD_COUNTER(MaxMemoryUsage);
-
- ADD_COUNTER(ErrorsCount);
-
+ };
+
+ ADD_COUNTER(Chunks);
+ ADD_COUNTER(Bytes);
+ ADD_COUNTER(RowsIn);
+ ADD_COUNTER(RowsOut);
+ ADD_COUNTER(MaxMemoryUsage);
+
+ ADD_COUNTER(ErrorsCount);
+
// if (stats.GetFinishTs() >= stats.GetStartTs()) {
// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
// }
- }
-
- for (const auto& stats : s.GetSinks()) {
- std::map<TString, TString> labels = {
- {"Task", ToString(taskId)},
+ }
+
+ for (const auto& stats : s.GetSinks()) {
+ std::map<TString, TString> labels = {
+ {"Task", ToString(taskId)},
{"Sink", ToString(stats.GetOutputIndex())}
- };
-
- ADD_COUNTER(Chunks)
- ADD_COUNTER(Bytes);
- ADD_COUNTER(RowsIn);
- ADD_COUNTER(RowsOut);
- ADD_COUNTER(MaxMemoryUsage);
-
- ADD_COUNTER(ErrorsCount);
-
+ };
+
+ ADD_COUNTER(Chunks)
+ ADD_COUNTER(Bytes);
+ ADD_COUNTER(RowsIn);
+ ADD_COUNTER(RowsOut);
+ ADD_COUNTER(MaxMemoryUsage);
+
+ ADD_COUNTER(ErrorsCount);
+
// if (stats.GetFinishTs() >= stats.GetStartTs()) {
// TaskStat.SetCounter(TaskStat.GetCounterName("TaskRunner", labels, "Total"), stats.GetFinishTs() - stats.GetStartTs());
// }
- }
-
+ }
+
#undef ADD_COUNTER
}
@@ -441,13 +441,13 @@ private:
YQL_LOG(DEBUG) << "Update channels";
for (const auto& [task, actorId] : Tasks) {
auto ev = MakeHolder<NDq::TEvDqCompute::TEvChannelsInfo>();
-
- for (const auto& input : task.GetInputs()) {
- for (const auto& channel : input.GetChannels()) {
- *ev->Record.AddUpdate() = channel;
- }
- }
-
+
+ for (const auto& input : task.GetInputs()) {
+ for (const auto& channel : input.GetChannels()) {
+ *ev->Record.AddUpdate() = channel;
+ }
+ }
+
for (const auto& output : task.GetOutputs()) {
for (const auto& channel : output.GetChannels()) {
*ev->Record.AddUpdate() = channel;
diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
index c4933a31fbb..f9ba89a33d9 100644
--- a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
+++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
@@ -9,7 +9,7 @@ TDqConfiguration::TDqConfiguration() {
REGISTER_SETTING(*this, MaxTasksPerOperation);
REGISTER_SETTING(*this, WorkersPerOperation);
REGISTER_SETTING(*this, MaxDataSizePerQuery);
- REGISTER_SETTING(*this, AnalyticsHopping);
+ REGISTER_SETTING(*this, AnalyticsHopping);
REGISTER_SETTING(*this, AnalyzeQuery);
REGISTER_SETTING(*this, _AnalyzeQueryPercentage);
REGISTER_SETTING(*this, MaxRetries);
diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.h b/ydb/library/yql/providers/dq/common/yql_dq_settings.h
index 7cb3642946a..3c37e4d48fd 100644
--- a/ydb/library/yql/providers/dq/common/yql_dq_settings.h
+++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.h
@@ -35,7 +35,7 @@ struct TDqSettings {
NCommon::TConfSetting<ui32, false> MaxTasksPerOperation;
NCommon::TConfSetting<ui32, false> WorkersPerOperation;
NCommon::TConfSetting<ui64, false> MaxDataSizePerQuery;
- NCommon::TConfSetting<bool, false> AnalyticsHopping;
+ NCommon::TConfSetting<bool, false> AnalyticsHopping;
NCommon::TConfSetting<bool, false> AnalyzeQuery;
NCommon::TConfSetting<int, false> _AnalyzeQueryPercentage;
NCommon::TConfSetting<int, false> MaxRetries;
@@ -78,7 +78,7 @@ struct TDqSettings {
s->SetValue(ToString(*this->name.Get())); \
}
- SAVE_SETTING(AnalyticsHopping);
+ SAVE_SETTING(AnalyticsHopping);
SAVE_SETTING(MaxRetries);
SAVE_SETTING(MaxNetworkRetries);
SAVE_SETTING(WorkersPerOperation);
diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
index 33e8883c7c4..9e87f81e7cf 100644
--- a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
+++ b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
@@ -15,35 +15,35 @@ using namespace NYql;
using namespace NYql::NDq;
using namespace NYql::NNodes;
-namespace {
-
-constexpr TStringBuf YQL_TIME = "_yql_time";
-
-TString BuildColumnName(const TExprBase column) {
- if (const auto columnName = column.Maybe<TCoAtom>()) {
- return columnName.Cast().StringValue();
- }
-
- if (const auto columnNames = column.Maybe<TCoAtomList>()) {
- TStringBuilder columnNameBuilder;
- for (const auto columnName : columnNames.Cast()) {
- columnNameBuilder.append(columnName.StringValue());
- columnNameBuilder.append("_");
- }
- return columnNameBuilder;
- }
-
- YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: "
- << column.Ptr()->Dump());
-}
-
-}
-
+namespace {
+
+constexpr TStringBuf YQL_TIME = "_yql_time";
+
+TString BuildColumnName(const TExprBase column) {
+ if (const auto columnName = column.Maybe<TCoAtom>()) {
+ return columnName.Cast().StringValue();
+ }
+
+ if (const auto columnNames = column.Maybe<TCoAtomList>()) {
+ TStringBuilder columnNameBuilder;
+ for (const auto columnName : columnNames.Cast()) {
+ columnNameBuilder.append(columnName.StringValue());
+ columnNameBuilder.append("_");
+ }
+ return columnNameBuilder;
+ }
+
+ YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: "
+ << column.Ptr()->Dump());
+}
+
+}
+
class TDqsLogicalOptProposalTransformer : public TOptimizeTransformerBase {
public:
- TDqsLogicalOptProposalTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config)
+ TDqsLogicalOptProposalTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config)
: TOptimizeTransformerBase(typeCtx, NLog::EComponent::ProviderDq, {})
- , Config(config)
+ , Config(config)
{
#define HNDL(name) "DqsLogical-"#name, Hndl(&TDqsLogicalOptProposalTransformer::name)
AddHandler(0, &TCoUnorderedBase::Match, HNDL(SkipUnordered));
@@ -74,15 +74,15 @@ protected:
TMaybeNode<TExprBase> RewriteAggregate(TExprBase node, TExprContext& ctx) {
auto aggregate = node.Cast<TCoAggregate>();
- auto input = aggregate.Input().Maybe<TDqConnection>();
-
- auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping");
- if (input) {
- if (hopSetting) {
- return RewriteAsHoppingWindow(node, ctx, input.Cast()).Cast();
- } else {
- return DqRewriteAggregate(node, ctx);
- }
+ auto input = aggregate.Input().Maybe<TDqConnection>();
+
+ auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping");
+ if (input) {
+ if (hopSetting) {
+ return RewriteAsHoppingWindow(node, ctx, input.Cast()).Cast();
+ } else {
+ return DqRewriteAggregate(node, ctx);
+ }
}
return node;
}
@@ -117,7 +117,7 @@ protected:
}
return node;
}
-
+
TMaybeNode<TExprBase> MergeQueriesWithSinks(TExprBase node, TExprContext& ctx) {
return DqMergeQueriesWithSinks(node, ctx);
}
@@ -126,661 +126,661 @@ protected:
return DqUnorderedInStage(node, TDqReadWrapBase::Match, ctx, Types);
}
-private:
- TMaybeNode<TExprBase> RewriteAsHoppingWindow(const TExprBase node, TExprContext& ctx, const TDqConnection& input) {
- const auto aggregate = node.Cast<TCoAggregate>();
- const auto pos = aggregate.Pos();
-
- YQL_CLOG(DEBUG, ProviderDq) << "OptimizeStreamingAggregate";
-
- EnsureNotDistinct(aggregate);
-
- const auto aggregateInputType = GetSeqItemType(node.Ptr()->Head().GetTypeAnn())->Cast<TStructExprType>();
- TKeysDescription keysDescription(*aggregateInputType, aggregate.Keys());
-
- if (keysDescription.NeedPickle()) {
- return Build<TCoMap>(ctx, pos)
- .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType))
- .Input<TCoAggregate>()
- .InitFrom(aggregate)
- .Input<TCoMap>()
- .Lambda(keysDescription.BuildPickleLambda(ctx, pos))
- .Input(input)
- .Build()
- .Build()
- .Done();
- }
-
- const auto maybeHopTraits = ExtractHopTraits(aggregate, ctx);
- if (!maybeHopTraits) {
- return nullptr;
- }
- const auto hopTraits = maybeHopTraits.Cast();
-
- const auto keyLambda = BuildKeySelector(pos, *aggregateInputType, aggregate.Keys().Ptr(), ctx);
- const auto timeExtractorLambda = BuildTimeExtractor(hopTraits, ctx);
- const auto initLambda = BuildInitHopLambda(aggregate, ctx);
- const auto updateLambda = BuildUpdateHopLambda(aggregate, ctx);
- const auto saveLambda = BuildSaveHopLambda(aggregate, ctx);
- const auto loadLambda = BuildLoadHopLambda(aggregate, ctx);
- const auto mergeLambda = BuildMergeHopLambda(aggregate, ctx);
- const auto finishLambda = BuildFinishHopLambda(aggregate, ctx);
-
- const auto streamArg = Build<TCoArgument>(ctx, pos).Name("stream").Done();
- auto multiHoppingCoreBuilder = Build<TCoMultiHoppingCore>(ctx, pos)
- .KeyExtractor(keyLambda)
- .TimeExtractor(timeExtractorLambda)
- .Hop(hopTraits.Hop())
- .Interval(hopTraits.Interval())
- .Delay(hopTraits.Delay())
- .DataWatermarks(hopTraits.DataWatermarks())
- .InitHandler(initLambda)
- .UpdateHandler(updateLambda)
- .MergeHandler(mergeLambda)
- .FinishHandler(finishLambda)
- .SaveHandler(saveLambda)
- .LoadHandler(loadLambda);
-
- if (Config->AnalyticsHopping.Get().GetOrElse(false)) {
- return Build<TCoPartitionsByKeys>(ctx, node.Pos())
- .Input(input.Ptr())
- .KeySelectorLambda(keyLambda)
- .SortDirections<TCoBool>()
- .Literal()
- .Value("true")
- .Build()
- .Build()
- .SortKeySelectorLambda(timeExtractorLambda)
- .ListHandlerLambda()
- .Args(streamArg)
- .Body<TCoForwardList>()
- .Stream(multiHoppingCoreBuilder
- .Input<TCoIterator>()
- .List(streamArg)
- .Build()
- .Done())
- .Build()
- .Build()
- .Done();
- } else {
- auto wrappedInput = input.Ptr();
- if (!aggregate.Keys().Empty()) {
- // Shuffle input connection by keys
- wrappedInput = WrapToShuffle(keysDescription, aggregate, input, ctx);
- if (!wrappedInput) {
- return nullptr;
- }
- }
-
- const auto stage = Build<TDqStage>(ctx, node.Pos())
- .Inputs()
- .Add(wrappedInput)
- .Build()
- .Program()
- .Args(streamArg)
- .Body<TCoMap>()
- .Input(multiHoppingCoreBuilder
- .Input<TCoFromFlow>()
- .Input(streamArg)
- .Build()
- .Done())
- .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType))
- .Build()
- .Build()
- .Settings(TDqStageSettings().BuildNode(ctx, node.Pos()))
- .Done();
-
- return Build<TDqCnUnionAll>(ctx, node.Pos())
- .Output()
- .Stage(stage)
- .Index().Build(0)
- .Build()
- .Done();
- }
- }
-
- TMaybeNode<TCoHoppingTraits> ExtractHopTraits(const TCoAggregate& aggregate, TExprContext& ctx) {
- const auto pos = aggregate.Pos();
-
- const auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping");
- if (!hopSetting) {
- ctx.AddError(TIssue(ctx.GetPosition(pos), "Aggregate over stream must have 'hopping' setting"));
- return TMaybeNode<TCoHoppingTraits>();
- }
-
- const auto maybeTraits = TMaybeNode<TCoHoppingTraits>(hopSetting->Child(1));
- if (!maybeTraits) {
- ctx.AddError(TIssue(ctx.GetPosition(pos), "Invalid 'hopping' setting in Aggregate"));
- return TMaybeNode<TCoHoppingTraits>();
- }
-
- const auto traits = maybeTraits.Cast();
-
- const auto checkIntervalParam = [&] (TExprBase param) -> ui64 {
- if (param.Maybe<TCoJust>()) {
- param = param.Cast<TCoJust>().Input();
- }
- if (!param.Maybe<TCoInterval>()) {
- ctx.AddError(TIssue(ctx.GetPosition(pos), "Not an interval data ctor"));
- return 0;
- }
- auto value = FromString<i64>(param.Cast<TCoInterval>().Literal().Value());
- if (value <= 0) {
- ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval value must be positive"));
- return 0;
- }
- return (ui64)value;
- };
-
- const auto hop = checkIntervalParam(traits.Hop());
- if (!hop) {
- return TMaybeNode<TCoHoppingTraits>();
- }
- const auto interval = checkIntervalParam(traits.Interval());
- if (!interval) {
- return TMaybeNode<TCoHoppingTraits>();
- }
- const auto delay = checkIntervalParam(traits.Delay());
- if (!delay) {
- return TMaybeNode<TCoHoppingTraits>();
- }
-
- if (interval < hop) {
- ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval must be greater or equal then hop"));
- return TMaybeNode<TCoHoppingTraits>();
- }
- if (delay < hop) {
- ctx.AddError(TIssue(ctx.GetPosition(pos), "Delay must be greater or equal then hop"));
- return TMaybeNode<TCoHoppingTraits>();
- }
-
- return Build<TCoHoppingTraits>(ctx, aggregate.Pos())
- .InitFrom(traits)
- .DataWatermarks(Config->AnalyticsHopping.Get().GetOrElse(false)
- ? ctx.NewAtom(aggregate.Pos(), "false")
- : traits.DataWatermarks().Ptr())
- .Done();
- }
-
- struct TKeysDescription {
- TVector<TString> PickleKeys;
- TVector<TString> MemberKeys;
-
- explicit TKeysDescription(const TStructExprType& rowType, const TCoAtomList& keys) {
- for (const auto& key : keys) {
- const auto index = rowType.FindItem(key.StringValue());
- Y_ENSURE(index);
-
- auto itemType = rowType.GetItems()[*index]->GetItemType();
- if (RemoveOptionalType(itemType)->GetKind() == ETypeAnnotationKind::Data) {
- MemberKeys.emplace_back(key.StringValue());
- continue;
- }
-
- PickleKeys.emplace_back(key.StringValue());
- }
- }
-
- TExprNode::TPtr BuildPickleLambda(TExprContext& ctx, TPositionHandle pos) const {
- TCoArgument arg = Build<TCoArgument>(ctx, pos)
- .Name("item")
- .Done();
-
- TExprBase body = arg;
-
- for (const auto& key : PickleKeys) {
- const auto member = Build<TCoMember>(ctx, pos)
- .Name().Build(key)
- .Struct(arg)
- .Done()
- .Ptr();
-
- body = Build<TCoReplaceMember>(ctx, pos)
- .Struct(body)
- .Name().Build(key)
- .Item(ctx.NewCallable(pos, "StablePickle", { member }))
- .Done();
- }
-
- return Build<TCoLambda>(ctx, pos)
- .Args({arg})
- .Body(body)
- .Done()
- .Ptr();
- }
-
- TExprNode::TPtr BuildUnpickleLambda(TExprContext& ctx, TPositionHandle pos, const TStructExprType& rowType) {
- TCoArgument arg = Build<TCoArgument>(ctx, pos)
- .Name("item")
- .Done();
-
- TExprBase body = arg;
-
- for (const auto& key : PickleKeys) {
- const auto index = rowType.FindItem(key);
- Y_ENSURE(index);
-
- auto itemType = rowType.GetItems().at(*index)->GetItemType();
- const auto member = Build<TCoMember>(ctx, pos)
- .Name().Build(key)
- .Struct(arg)
- .Done()
- .Ptr();
-
- body = Build<TCoReplaceMember>(ctx, pos)
- .Struct(body)
- .Name().Build(key)
- .Item(ctx.NewCallable(pos, "Unpickle", { ExpandType(pos, *itemType, ctx), member }))
- .Done();
- }
-
- return Build<TCoLambda>(ctx, pos)
- .Args({arg})
- .Body(body)
- .Done()
- .Ptr();
- }
-
- TVector<TCoAtom> GetKeysList(TExprContext& ctx, TPositionHandle pos) const {
- TVector<TCoAtom> res;
- res.reserve(PickleKeys.size() + MemberKeys.size());
-
- for (const auto& pickleKey : PickleKeys) {
- res.emplace_back(Build<TCoAtom>(ctx, pos).Value(pickleKey).Done());
- }
- for (const auto& memberKey : MemberKeys) {
- res.emplace_back(Build<TCoAtom>(ctx, pos).Value(memberKey).Done());
- }
- return res;
- }
-
- bool NeedPickle() const {
- return !PickleKeys.empty();
- }
- };
-
- TExprNode::TPtr WrapToShuffle(
- const TKeysDescription& keysDescription,
- const TCoAggregate& aggregate,
- const TDqConnection& input,
- TExprContext& ctx)
- {
- auto pos = aggregate.Pos();
-
- TDqStageBase mappedInput = input.Output().Stage();
- if (keysDescription.NeedPickle()) {
- mappedInput = Build<TDqStage>(ctx, pos)
- .Inputs()
- .Add<TDqCnMap>()
- .Output()
- .Stage(input.Output().Stage())
- .Index(input.Output().Index())
- .Build()
- .Build()
- .Build()
- .Program()
- .Args({"stream"})
- .Body<TCoMap>()
- .Input("stream")
- .Lambda(keysDescription.BuildPickleLambda(ctx, pos))
- .Build()
- .Build()
- .Settings(TDqStageSettings().BuildNode(ctx, pos))
- .Done();
- }
-
+private:
+ TMaybeNode<TExprBase> RewriteAsHoppingWindow(const TExprBase node, TExprContext& ctx, const TDqConnection& input) {
+ const auto aggregate = node.Cast<TCoAggregate>();
+ const auto pos = aggregate.Pos();
+
+ YQL_CLOG(DEBUG, ProviderDq) << "OptimizeStreamingAggregate";
+
+ EnsureNotDistinct(aggregate);
+
+ const auto aggregateInputType = GetSeqItemType(node.Ptr()->Head().GetTypeAnn())->Cast<TStructExprType>();
+ TKeysDescription keysDescription(*aggregateInputType, aggregate.Keys());
+
+ if (keysDescription.NeedPickle()) {
+ return Build<TCoMap>(ctx, pos)
+ .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType))
+ .Input<TCoAggregate>()
+ .InitFrom(aggregate)
+ .Input<TCoMap>()
+ .Lambda(keysDescription.BuildPickleLambda(ctx, pos))
+ .Input(input)
+ .Build()
+ .Build()
+ .Done();
+ }
+
+ const auto maybeHopTraits = ExtractHopTraits(aggregate, ctx);
+ if (!maybeHopTraits) {
+ return nullptr;
+ }
+ const auto hopTraits = maybeHopTraits.Cast();
+
+ const auto keyLambda = BuildKeySelector(pos, *aggregateInputType, aggregate.Keys().Ptr(), ctx);
+ const auto timeExtractorLambda = BuildTimeExtractor(hopTraits, ctx);
+ const auto initLambda = BuildInitHopLambda(aggregate, ctx);
+ const auto updateLambda = BuildUpdateHopLambda(aggregate, ctx);
+ const auto saveLambda = BuildSaveHopLambda(aggregate, ctx);
+ const auto loadLambda = BuildLoadHopLambda(aggregate, ctx);
+ const auto mergeLambda = BuildMergeHopLambda(aggregate, ctx);
+ const auto finishLambda = BuildFinishHopLambda(aggregate, ctx);
+
+ const auto streamArg = Build<TCoArgument>(ctx, pos).Name("stream").Done();
+ auto multiHoppingCoreBuilder = Build<TCoMultiHoppingCore>(ctx, pos)
+ .KeyExtractor(keyLambda)
+ .TimeExtractor(timeExtractorLambda)
+ .Hop(hopTraits.Hop())
+ .Interval(hopTraits.Interval())
+ .Delay(hopTraits.Delay())
+ .DataWatermarks(hopTraits.DataWatermarks())
+ .InitHandler(initLambda)
+ .UpdateHandler(updateLambda)
+ .MergeHandler(mergeLambda)
+ .FinishHandler(finishLambda)
+ .SaveHandler(saveLambda)
+ .LoadHandler(loadLambda);
+
+ if (Config->AnalyticsHopping.Get().GetOrElse(false)) {
+ return Build<TCoPartitionsByKeys>(ctx, node.Pos())
+ .Input(input.Ptr())
+ .KeySelectorLambda(keyLambda)
+ .SortDirections<TCoBool>()
+ .Literal()
+ .Value("true")
+ .Build()
+ .Build()
+ .SortKeySelectorLambda(timeExtractorLambda)
+ .ListHandlerLambda()
+ .Args(streamArg)
+ .Body<TCoForwardList>()
+ .Stream(multiHoppingCoreBuilder
+ .Input<TCoIterator>()
+ .List(streamArg)
+ .Build()
+ .Done())
+ .Build()
+ .Build()
+ .Done();
+ } else {
+ auto wrappedInput = input.Ptr();
+ if (!aggregate.Keys().Empty()) {
+ // Shuffle input connection by keys
+ wrappedInput = WrapToShuffle(keysDescription, aggregate, input, ctx);
+ if (!wrappedInput) {
+ return nullptr;
+ }
+ }
+
+ const auto stage = Build<TDqStage>(ctx, node.Pos())
+ .Inputs()
+ .Add(wrappedInput)
+ .Build()
+ .Program()
+ .Args(streamArg)
+ .Body<TCoMap>()
+ .Input(multiHoppingCoreBuilder
+ .Input<TCoFromFlow>()
+ .Input(streamArg)
+ .Build()
+ .Done())
+ .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType))
+ .Build()
+ .Build()
+ .Settings(TDqStageSettings().BuildNode(ctx, node.Pos()))
+ .Done();
+
+ return Build<TDqCnUnionAll>(ctx, node.Pos())
+ .Output()
+ .Stage(stage)
+ .Index().Build(0)
+ .Build()
+ .Done();
+ }
+ }
+
+ TMaybeNode<TCoHoppingTraits> ExtractHopTraits(const TCoAggregate& aggregate, TExprContext& ctx) {
+ const auto pos = aggregate.Pos();
+
+ const auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping");
+ if (!hopSetting) {
+ ctx.AddError(TIssue(ctx.GetPosition(pos), "Aggregate over stream must have 'hopping' setting"));
+ return TMaybeNode<TCoHoppingTraits>();
+ }
+
+ const auto maybeTraits = TMaybeNode<TCoHoppingTraits>(hopSetting->Child(1));
+ if (!maybeTraits) {
+ ctx.AddError(TIssue(ctx.GetPosition(pos), "Invalid 'hopping' setting in Aggregate"));
+ return TMaybeNode<TCoHoppingTraits>();
+ }
+
+ const auto traits = maybeTraits.Cast();
+
+ const auto checkIntervalParam = [&] (TExprBase param) -> ui64 {
+ if (param.Maybe<TCoJust>()) {
+ param = param.Cast<TCoJust>().Input();
+ }
+ if (!param.Maybe<TCoInterval>()) {
+ ctx.AddError(TIssue(ctx.GetPosition(pos), "Not an interval data ctor"));
+ return 0;
+ }
+ auto value = FromString<i64>(param.Cast<TCoInterval>().Literal().Value());
+ if (value <= 0) {
+ ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval value must be positive"));
+ return 0;
+ }
+ return (ui64)value;
+ };
+
+ const auto hop = checkIntervalParam(traits.Hop());
+ if (!hop) {
+ return TMaybeNode<TCoHoppingTraits>();
+ }
+ const auto interval = checkIntervalParam(traits.Interval());
+ if (!interval) {
+ return TMaybeNode<TCoHoppingTraits>();
+ }
+ const auto delay = checkIntervalParam(traits.Delay());
+ if (!delay) {
+ return TMaybeNode<TCoHoppingTraits>();
+ }
+
+ if (interval < hop) {
+ ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval must be greater or equal then hop"));
+ return TMaybeNode<TCoHoppingTraits>();
+ }
+ if (delay < hop) {
+ ctx.AddError(TIssue(ctx.GetPosition(pos), "Delay must be greater or equal then hop"));
+ return TMaybeNode<TCoHoppingTraits>();
+ }
+
+ return Build<TCoHoppingTraits>(ctx, aggregate.Pos())
+ .InitFrom(traits)
+ .DataWatermarks(Config->AnalyticsHopping.Get().GetOrElse(false)
+ ? ctx.NewAtom(aggregate.Pos(), "false")
+ : traits.DataWatermarks().Ptr())
+ .Done();
+ }
+
+ struct TKeysDescription {
+ TVector<TString> PickleKeys;
+ TVector<TString> MemberKeys;
+
+ explicit TKeysDescription(const TStructExprType& rowType, const TCoAtomList& keys) {
+ for (const auto& key : keys) {
+ const auto index = rowType.FindItem(key.StringValue());
+ Y_ENSURE(index);
+
+ auto itemType = rowType.GetItems()[*index]->GetItemType();
+ if (RemoveOptionalType(itemType)->GetKind() == ETypeAnnotationKind::Data) {
+ MemberKeys.emplace_back(key.StringValue());
+ continue;
+ }
+
+ PickleKeys.emplace_back(key.StringValue());
+ }
+ }
+
+ TExprNode::TPtr BuildPickleLambda(TExprContext& ctx, TPositionHandle pos) const {
+ TCoArgument arg = Build<TCoArgument>(ctx, pos)
+ .Name("item")
+ .Done();
+
+ TExprBase body = arg;
+
+ for (const auto& key : PickleKeys) {
+ const auto member = Build<TCoMember>(ctx, pos)
+ .Name().Build(key)
+ .Struct(arg)
+ .Done()
+ .Ptr();
+
+ body = Build<TCoReplaceMember>(ctx, pos)
+ .Struct(body)
+ .Name().Build(key)
+ .Item(ctx.NewCallable(pos, "StablePickle", { member }))
+ .Done();
+ }
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({arg})
+ .Body(body)
+ .Done()
+ .Ptr();
+ }
+
+ TExprNode::TPtr BuildUnpickleLambda(TExprContext& ctx, TPositionHandle pos, const TStructExprType& rowType) {
+ TCoArgument arg = Build<TCoArgument>(ctx, pos)
+ .Name("item")
+ .Done();
+
+ TExprBase body = arg;
+
+ for (const auto& key : PickleKeys) {
+ const auto index = rowType.FindItem(key);
+ Y_ENSURE(index);
+
+ auto itemType = rowType.GetItems().at(*index)->GetItemType();
+ const auto member = Build<TCoMember>(ctx, pos)
+ .Name().Build(key)
+ .Struct(arg)
+ .Done()
+ .Ptr();
+
+ body = Build<TCoReplaceMember>(ctx, pos)
+ .Struct(body)
+ .Name().Build(key)
+ .Item(ctx.NewCallable(pos, "Unpickle", { ExpandType(pos, *itemType, ctx), member }))
+ .Done();
+ }
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({arg})
+ .Body(body)
+ .Done()
+ .Ptr();
+ }
+
+ TVector<TCoAtom> GetKeysList(TExprContext& ctx, TPositionHandle pos) const {
+ TVector<TCoAtom> res;
+ res.reserve(PickleKeys.size() + MemberKeys.size());
+
+ for (const auto& pickleKey : PickleKeys) {
+ res.emplace_back(Build<TCoAtom>(ctx, pos).Value(pickleKey).Done());
+ }
+ for (const auto& memberKey : MemberKeys) {
+ res.emplace_back(Build<TCoAtom>(ctx, pos).Value(memberKey).Done());
+ }
+ return res;
+ }
+
+ bool NeedPickle() const {
+ return !PickleKeys.empty();
+ }
+ };
+
+ TExprNode::TPtr WrapToShuffle(
+ const TKeysDescription& keysDescription,
+ const TCoAggregate& aggregate,
+ const TDqConnection& input,
+ TExprContext& ctx)
+ {
+ auto pos = aggregate.Pos();
+
+ TDqStageBase mappedInput = input.Output().Stage();
+ if (keysDescription.NeedPickle()) {
+ mappedInput = Build<TDqStage>(ctx, pos)
+ .Inputs()
+ .Add<TDqCnMap>()
+ .Output()
+ .Stage(input.Output().Stage())
+ .Index(input.Output().Index())
+ .Build()
+ .Build()
+ .Build()
+ .Program()
+ .Args({"stream"})
+ .Body<TCoMap>()
+ .Input("stream")
+ .Lambda(keysDescription.BuildPickleLambda(ctx, pos))
+ .Build()
+ .Build()
+ .Settings(TDqStageSettings().BuildNode(ctx, pos))
+ .Done();
+ }
+
return Build<TDqCnHashShuffle>(ctx, pos)
- .Output()
- .Stage(mappedInput)
- .Index().Value("0").Build()
- .Build()
- .KeyColumns()
- .Add(keysDescription.GetKeysList(ctx, pos))
- .Build()
- .Done()
- .Ptr();
- }
-
- void EnsureNotDistinct(const TCoAggregate& aggregate) {
- const auto& aggregateHandlers = aggregate.Handlers();
-
- YQL_ENSURE(
- AllOf(aggregateHandlers, [](const auto& t){ return !t.DistinctName(); }),
- "Distinct is not supported for aggregation with hop");
- }
-
- TExprNode::TPtr BuildTimeExtractor(const TCoHoppingTraits& hoppingTraits, TExprContext& ctx) {
- const auto pos = hoppingTraits.Pos();
-
- if (hoppingTraits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>()->GetSize() == 0) {
- // The case when no fields are used in lambda. F.e. when it has only DependsOn.
- return ctx.DeepCopyLambda(hoppingTraits.TimeExtractor().Ref());
- }
-
- return Build<TCoLambda>(ctx, pos)
- .Args({"item"})
- .Body<TExprApplier>()
- .Apply(hoppingTraits.TimeExtractor())
- .With<TCoSafeCast>(0)
- .Type(hoppingTraits.ItemType())
- .Value("item")
- .Build()
- .Build()
- .Done()
- .Ptr();
- }
-
- TExprNode::TPtr BuildInitHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
- const auto pos = aggregate.Pos();
- const auto& aggregateHandlers = aggregate.Handlers();
-
- const auto initItemArg = Build<TCoArgument>(ctx, pos).Name("item").Done();
-
- TVector<TExprBase> structItems;
- structItems.reserve(aggregateHandlers.Size());
-
- ui32 index = 0;
- for (const auto& handler : aggregateHandlers) {
- const auto tuple = handler.Cast<TCoAggregateTuple>();
-
- TMaybeNode<TExprBase> applier;
- if (tuple.Trait().InitHandler().Args().Size() == 1) {
- applier = Build<TExprApplier>(ctx, pos)
- .Apply(tuple.Trait().InitHandler())
- .With(0, initItemArg)
- .Done();
- } else {
- applier = Build<TExprApplier>(ctx, pos)
- .Apply(tuple.Trait().InitHandler())
- .With(0, initItemArg)
- .With<TCoUint32>(1)
- .Literal().Build(ToString(index))
- .Build()
- .Done();
- }
-
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(BuildColumnName(tuple.ColumnName()))
- .Value(applier)
- .Done());
- ++index;
- }
-
- return Build<TCoLambda>(ctx, pos)
- .Args({initItemArg})
- .Body<TCoAsStruct>()
- .Add(structItems)
- .Build()
- .Done()
- .Ptr();
- }
-
- TExprNode::TPtr BuildUpdateHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
- const auto pos = aggregate.Pos();
- const auto aggregateHandlers = aggregate.Handlers();
-
- const auto updateItemArg = Build<TCoArgument>(ctx, pos).Name("item").Done();
- const auto updateStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done();
-
- TVector<TExprBase> structItems;
- structItems.reserve(aggregateHandlers.Size());
-
- i32 index = 0;
- for (const auto& handler : aggregateHandlers) {
- const auto tuple = handler.Cast<TCoAggregateTuple>();
- const TString columnName = BuildColumnName(tuple.ColumnName());
-
- const auto member = Build<TCoMember>(ctx, pos)
- .Struct(updateStateArg)
- .Name().Build(columnName)
- .Done();
-
- TMaybeNode<TExprBase> applier;
- if (tuple.Trait().UpdateHandler().Args().Size() == 2) {
- applier = Build<TExprApplier>(ctx, pos)
- .Apply(tuple.Trait().UpdateHandler())
- .With(0, updateItemArg)
- .With(1, member)
- .Done();
- } else {
- applier = Build<TExprApplier>(ctx, pos)
- .Apply(tuple.Trait().UpdateHandler())
- .With(0, updateItemArg)
- .With(1, member)
- .With<TCoUint32>(2)
- .Literal().Build(ToString(index))
- .Build()
- .Done();
- }
-
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(columnName)
- .Value(applier)
- .Done());
- ++index;
- }
-
- return Build<TCoLambda>(ctx, pos)
- .Args({updateItemArg, updateStateArg})
- .Body<TCoAsStruct>()
- .Add(structItems)
- .Build()
- .Done()
- .Ptr();
- }
-
- TExprNode::TPtr BuildMergeHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
- const auto pos = aggregate.Pos();
- const auto& aggregateHandlers = aggregate.Handlers();
-
- const auto mergeState1Arg = Build<TCoArgument>(ctx, pos).Name("state1").Done();
- const auto mergeState2Arg = Build<TCoArgument>(ctx, pos).Name("state2").Done();
-
- TVector<TExprBase> structItems;
- structItems.reserve(aggregateHandlers.Size());
-
- for (const auto& handler : aggregateHandlers) {
- const auto tuple = handler.Cast<TCoAggregateTuple>();
- const TString columnName = BuildColumnName(tuple.ColumnName());
-
- const auto member1 = Build<TCoMember>(ctx, pos)
- .Struct(mergeState1Arg)
- .Name().Build(columnName)
- .Done();
- const auto member2 = Build<TCoMember>(ctx, pos)
- .Struct(mergeState2Arg)
- .Name().Build(columnName)
- .Done();
-
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(columnName)
- .Value<TExprApplier>()
- .Apply(tuple.Trait().MergeHandler())
- .With(0, member1)
- .With(1, member2)
- .Build()
- .Done());
- }
-
- return Build<TCoLambda>(ctx, pos)
- .Args({mergeState1Arg, mergeState2Arg})
- .Body<TCoAsStruct>()
- .Add(structItems)
- .Build()
- .Done()
- .Ptr();
- }
-
- TExprNode::TPtr BuildFinishHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
- const auto pos = aggregate.Pos();
- const auto keyColumns = aggregate.Keys();
- const auto aggregateHandlers = aggregate.Handlers();
-
- const auto finishKeyArg = Build<TCoArgument>(ctx, pos).Name("key").Done();
- const auto finishStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done();
- const auto finishTimeArg = Build<TCoArgument>(ctx, pos).Name("time").Done();
-
- TVector<TExprBase> structItems;
- structItems.reserve(keyColumns.Size() + aggregateHandlers.Size() + 1);
-
- if (keyColumns.Size() == 1) {
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name(keyColumns.Item(0))
- .Value(finishKeyArg)
- .Done());
- } else {
- for (size_t i = 0; i < keyColumns.Size(); ++i) {
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name(keyColumns.Item(i))
- .Value<TCoNth>()
- .Tuple(finishKeyArg)
- .Index<TCoAtom>()
- .Value(ToString(i))
- .Build()
- .Build()
- .Done());
- }
- }
-
- for (const auto& handler : aggregateHandlers) {
- const auto tuple = handler.Cast<TCoAggregateTuple>();
- const TString compoundColumnName = BuildColumnName(tuple.ColumnName());
-
- const auto member = Build<TCoMember>(ctx, pos)
- .Struct(finishStateArg)
- .Name().Build(compoundColumnName)
- .Done();
-
- if (tuple.ColumnName().Maybe<TCoAtom>()) {
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(compoundColumnName)
- .Value<TExprApplier>()
- .Apply(tuple.Trait().FinishHandler())
- .With(0, member)
- .Build()
- .Done());
-
- continue;
- }
-
- if (const auto namesList = tuple.ColumnName().Maybe<TCoAtomList>()) {
- const auto expApplier = Build<TExprApplier>(ctx, pos)
- .Apply(tuple.Trait().FinishHandler())
- .With(0, member)
- .Done();
-
- int index = 0;
- for (const auto columnName : namesList.Cast()) {
- const auto extracter = Build<TCoNth>(ctx, pos)
- .Tuple(expApplier)
- .Index<TCoAtom>().Build(index++)
- .Done();
-
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name(columnName)
- .Value(extracter)
- .Done());
- }
-
- continue;
- }
-
- YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: "
- << tuple.ColumnName().Ptr()->Dump());
- }
-
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(YQL_TIME)
- .Value(finishTimeArg)
- .Done());
-
- return Build<TCoLambda>(ctx, pos)
- .Args({finishKeyArg, finishStateArg, finishTimeArg})
- .Body<TCoAsStruct>()
- .Add(structItems)
- .Build()
- .Done()
- .Ptr();
- }
-
- TExprNode::TPtr BuildSaveHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
- const auto pos = aggregate.Pos();
- const auto aggregateHandlers = aggregate.Handlers();
-
- const auto saveStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done();
-
- TVector<TExprBase> structItems;
- structItems.reserve(aggregateHandlers.Size());
-
- for (const auto& handler : aggregateHandlers) {
- const auto tuple = handler.Cast<TCoAggregateTuple>();
- const TString columnName = BuildColumnName(tuple.ColumnName());
-
- const auto member = Build<TCoMember>(ctx, pos)
- .Struct(saveStateArg)
- .Name().Build(columnName)
- .Done();
-
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(columnName)
- .Value<TExprApplier>()
- .Apply(tuple.Trait().SaveHandler())
- .With(0, member)
- .Build()
- .Done());
- }
-
- return Build<TCoLambda>(ctx, pos)
- .Args({saveStateArg})
- .Body<TCoAsStruct>()
- .Add(structItems)
- .Build()
- .Done()
- .Ptr();
- }
-
- TExprNode::TPtr BuildLoadHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
- const auto pos = aggregate.Pos();
- const auto aggregateHandlers = aggregate.Handlers();
-
- TCoArgument loadStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done();
-
- TVector<TExprBase> structItems;
- structItems.reserve(aggregateHandlers.Size());
-
- for (const auto& handler : aggregateHandlers) {
- const auto tuple = handler.Cast<TCoAggregateTuple>();
- const TString columnName = BuildColumnName(tuple.ColumnName());
-
- const auto member = Build<TCoMember>(ctx, pos)
- .Struct(loadStateArg)
- .Name().Build(columnName)
- .Done();
-
- structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
- .Name().Build(columnName)
- .Value<TExprApplier>()
- .Apply(tuple.Trait().LoadHandler())
- .With(0, member)
- .Build()
- .Done());
- }
-
- return Build<TCoLambda>(ctx, pos)
- .Args({loadStateArg})
- .Body<TCoAsStruct>()
- .Add(structItems)
- .Build()
- .Done()
- .Ptr();
- }
-
-private:
- TDqConfiguration::TPtr Config;
+ .Output()
+ .Stage(mappedInput)
+ .Index().Value("0").Build()
+ .Build()
+ .KeyColumns()
+ .Add(keysDescription.GetKeysList(ctx, pos))
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+ void EnsureNotDistinct(const TCoAggregate& aggregate) {
+ const auto& aggregateHandlers = aggregate.Handlers();
+
+ YQL_ENSURE(
+ AllOf(aggregateHandlers, [](const auto& t){ return !t.DistinctName(); }),
+ "Distinct is not supported for aggregation with hop");
+ }
+
+ TExprNode::TPtr BuildTimeExtractor(const TCoHoppingTraits& hoppingTraits, TExprContext& ctx) {
+ const auto pos = hoppingTraits.Pos();
+
+ if (hoppingTraits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>()->GetSize() == 0) {
+ // The case when no fields are used in lambda. F.e. when it has only DependsOn.
+ return ctx.DeepCopyLambda(hoppingTraits.TimeExtractor().Ref());
+ }
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({"item"})
+ .Body<TExprApplier>()
+ .Apply(hoppingTraits.TimeExtractor())
+ .With<TCoSafeCast>(0)
+ .Type(hoppingTraits.ItemType())
+ .Value("item")
+ .Build()
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+ TExprNode::TPtr BuildInitHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
+ const auto pos = aggregate.Pos();
+ const auto& aggregateHandlers = aggregate.Handlers();
+
+ const auto initItemArg = Build<TCoArgument>(ctx, pos).Name("item").Done();
+
+ TVector<TExprBase> structItems;
+ structItems.reserve(aggregateHandlers.Size());
+
+ ui32 index = 0;
+ for (const auto& handler : aggregateHandlers) {
+ const auto tuple = handler.Cast<TCoAggregateTuple>();
+
+ TMaybeNode<TExprBase> applier;
+ if (tuple.Trait().InitHandler().Args().Size() == 1) {
+ applier = Build<TExprApplier>(ctx, pos)
+ .Apply(tuple.Trait().InitHandler())
+ .With(0, initItemArg)
+ .Done();
+ } else {
+ applier = Build<TExprApplier>(ctx, pos)
+ .Apply(tuple.Trait().InitHandler())
+ .With(0, initItemArg)
+ .With<TCoUint32>(1)
+ .Literal().Build(ToString(index))
+ .Build()
+ .Done();
+ }
+
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(BuildColumnName(tuple.ColumnName()))
+ .Value(applier)
+ .Done());
+ ++index;
+ }
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({initItemArg})
+ .Body<TCoAsStruct>()
+ .Add(structItems)
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+ TExprNode::TPtr BuildUpdateHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
+ const auto pos = aggregate.Pos();
+ const auto aggregateHandlers = aggregate.Handlers();
+
+ const auto updateItemArg = Build<TCoArgument>(ctx, pos).Name("item").Done();
+ const auto updateStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done();
+
+ TVector<TExprBase> structItems;
+ structItems.reserve(aggregateHandlers.Size());
+
+ i32 index = 0;
+ for (const auto& handler : aggregateHandlers) {
+ const auto tuple = handler.Cast<TCoAggregateTuple>();
+ const TString columnName = BuildColumnName(tuple.ColumnName());
+
+ const auto member = Build<TCoMember>(ctx, pos)
+ .Struct(updateStateArg)
+ .Name().Build(columnName)
+ .Done();
+
+ TMaybeNode<TExprBase> applier;
+ if (tuple.Trait().UpdateHandler().Args().Size() == 2) {
+ applier = Build<TExprApplier>(ctx, pos)
+ .Apply(tuple.Trait().UpdateHandler())
+ .With(0, updateItemArg)
+ .With(1, member)
+ .Done();
+ } else {
+ applier = Build<TExprApplier>(ctx, pos)
+ .Apply(tuple.Trait().UpdateHandler())
+ .With(0, updateItemArg)
+ .With(1, member)
+ .With<TCoUint32>(2)
+ .Literal().Build(ToString(index))
+ .Build()
+ .Done();
+ }
+
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(columnName)
+ .Value(applier)
+ .Done());
+ ++index;
+ }
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({updateItemArg, updateStateArg})
+ .Body<TCoAsStruct>()
+ .Add(structItems)
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+ TExprNode::TPtr BuildMergeHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
+ const auto pos = aggregate.Pos();
+ const auto& aggregateHandlers = aggregate.Handlers();
+
+ const auto mergeState1Arg = Build<TCoArgument>(ctx, pos).Name("state1").Done();
+ const auto mergeState2Arg = Build<TCoArgument>(ctx, pos).Name("state2").Done();
+
+ TVector<TExprBase> structItems;
+ structItems.reserve(aggregateHandlers.Size());
+
+ for (const auto& handler : aggregateHandlers) {
+ const auto tuple = handler.Cast<TCoAggregateTuple>();
+ const TString columnName = BuildColumnName(tuple.ColumnName());
+
+ const auto member1 = Build<TCoMember>(ctx, pos)
+ .Struct(mergeState1Arg)
+ .Name().Build(columnName)
+ .Done();
+ const auto member2 = Build<TCoMember>(ctx, pos)
+ .Struct(mergeState2Arg)
+ .Name().Build(columnName)
+ .Done();
+
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(columnName)
+ .Value<TExprApplier>()
+ .Apply(tuple.Trait().MergeHandler())
+ .With(0, member1)
+ .With(1, member2)
+ .Build()
+ .Done());
+ }
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({mergeState1Arg, mergeState2Arg})
+ .Body<TCoAsStruct>()
+ .Add(structItems)
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+ TExprNode::TPtr BuildFinishHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
+ const auto pos = aggregate.Pos();
+ const auto keyColumns = aggregate.Keys();
+ const auto aggregateHandlers = aggregate.Handlers();
+
+ const auto finishKeyArg = Build<TCoArgument>(ctx, pos).Name("key").Done();
+ const auto finishStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done();
+ const auto finishTimeArg = Build<TCoArgument>(ctx, pos).Name("time").Done();
+
+ TVector<TExprBase> structItems;
+ structItems.reserve(keyColumns.Size() + aggregateHandlers.Size() + 1);
+
+ if (keyColumns.Size() == 1) {
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name(keyColumns.Item(0))
+ .Value(finishKeyArg)
+ .Done());
+ } else {
+ for (size_t i = 0; i < keyColumns.Size(); ++i) {
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name(keyColumns.Item(i))
+ .Value<TCoNth>()
+ .Tuple(finishKeyArg)
+ .Index<TCoAtom>()
+ .Value(ToString(i))
+ .Build()
+ .Build()
+ .Done());
+ }
+ }
+
+ for (const auto& handler : aggregateHandlers) {
+ const auto tuple = handler.Cast<TCoAggregateTuple>();
+ const TString compoundColumnName = BuildColumnName(tuple.ColumnName());
+
+ const auto member = Build<TCoMember>(ctx, pos)
+ .Struct(finishStateArg)
+ .Name().Build(compoundColumnName)
+ .Done();
+
+ if (tuple.ColumnName().Maybe<TCoAtom>()) {
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(compoundColumnName)
+ .Value<TExprApplier>()
+ .Apply(tuple.Trait().FinishHandler())
+ .With(0, member)
+ .Build()
+ .Done());
+
+ continue;
+ }
+
+ if (const auto namesList = tuple.ColumnName().Maybe<TCoAtomList>()) {
+ const auto expApplier = Build<TExprApplier>(ctx, pos)
+ .Apply(tuple.Trait().FinishHandler())
+ .With(0, member)
+ .Done();
+
+ int index = 0;
+ for (const auto columnName : namesList.Cast()) {
+ const auto extracter = Build<TCoNth>(ctx, pos)
+ .Tuple(expApplier)
+ .Index<TCoAtom>().Build(index++)
+ .Done();
+
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name(columnName)
+ .Value(extracter)
+ .Done());
+ }
+
+ continue;
+ }
+
+ YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: "
+ << tuple.ColumnName().Ptr()->Dump());
+ }
+
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(YQL_TIME)
+ .Value(finishTimeArg)
+ .Done());
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({finishKeyArg, finishStateArg, finishTimeArg})
+ .Body<TCoAsStruct>()
+ .Add(structItems)
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+ TExprNode::TPtr BuildSaveHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
+ const auto pos = aggregate.Pos();
+ const auto aggregateHandlers = aggregate.Handlers();
+
+ const auto saveStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done();
+
+ TVector<TExprBase> structItems;
+ structItems.reserve(aggregateHandlers.Size());
+
+ for (const auto& handler : aggregateHandlers) {
+ const auto tuple = handler.Cast<TCoAggregateTuple>();
+ const TString columnName = BuildColumnName(tuple.ColumnName());
+
+ const auto member = Build<TCoMember>(ctx, pos)
+ .Struct(saveStateArg)
+ .Name().Build(columnName)
+ .Done();
+
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(columnName)
+ .Value<TExprApplier>()
+ .Apply(tuple.Trait().SaveHandler())
+ .With(0, member)
+ .Build()
+ .Done());
+ }
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({saveStateArg})
+ .Body<TCoAsStruct>()
+ .Add(structItems)
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+ TExprNode::TPtr BuildLoadHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) {
+ const auto pos = aggregate.Pos();
+ const auto aggregateHandlers = aggregate.Handlers();
+
+ TCoArgument loadStateArg = Build<TCoArgument>(ctx, pos).Name("state").Done();
+
+ TVector<TExprBase> structItems;
+ structItems.reserve(aggregateHandlers.Size());
+
+ for (const auto& handler : aggregateHandlers) {
+ const auto tuple = handler.Cast<TCoAggregateTuple>();
+ const TString columnName = BuildColumnName(tuple.ColumnName());
+
+ const auto member = Build<TCoMember>(ctx, pos)
+ .Struct(loadStateArg)
+ .Name().Build(columnName)
+ .Done();
+
+ structItems.push_back(Build<TCoNameValueTuple>(ctx, pos)
+ .Name().Build(columnName)
+ .Value<TExprApplier>()
+ .Apply(tuple.Trait().LoadHandler())
+ .With(0, member)
+ .Build()
+ .Done());
+ }
+
+ return Build<TCoLambda>(ctx, pos)
+ .Args({loadStateArg})
+ .Body<TCoAsStruct>()
+ .Add(structItems)
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+private:
+ TDqConfiguration::TPtr Config;
};
-THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config) {
- return THolder(new TDqsLogicalOptProposalTransformer(typeCtx, config));
+THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config) {
+ return THolder(new TDqsLogicalOptProposalTransformer(typeCtx, config));
}
} // NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.h b/ydb/library/yql/providers/dq/opt/logical_optimize.h
index 13eb9aff9d8..54352ac344e 100644
--- a/ydb/library/yql/providers/dq/opt/logical_optimize.h
+++ b/ydb/library/yql/providers/dq/opt/logical_optimize.h
@@ -8,6 +8,6 @@
namespace NYql::NDqs {
-THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config);
+THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config);
} // namespace NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
index 42ae97ad833..8759f442219 100644
--- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
+++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
@@ -647,9 +647,9 @@ private:
TUploadList uploadList;
auto lambdaResult = GetLambda(&lambda, &untrustedUdfFlag, &level, &uploadList, result, ctx, hasGraphParams);
if (lambdaResult.first.Level == TStatus::Error) {
- if (State->Settings->FallbackPolicy.Get().GetOrElse("default") == "never" || State->TypeCtx->ForceDq) {
- return SyncError();
- }
+ if (State->Settings->FallbackPolicy.Get().GetOrElse("default") == "never" || State->TypeCtx->ForceDq) {
+ return SyncError();
+ }
return Fallback();
}
if (lambdaResult.first.Level != TStatus::Ok) {
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp
index e8eb1ece76c..a8b5e598a08 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp
@@ -27,7 +27,7 @@ class TDqDataProviderSink: public TDataProviderBase {
public:
TDqDataProviderSink(const TDqStatePtr& state)
: State(state)
- , LogOptTransformer([state] () { return CreateDqsLogOptTransformer(/*TODO: State->TypeCtx);*/nullptr, state->Settings); })
+ , LogOptTransformer([state] () { return CreateDqsLogOptTransformer(/*TODO: State->TypeCtx);*/nullptr, state->Settings); })
, PhyOptTransformer([] () { return CreateDqsPhyOptTransformer(/*TODO: State->TypeCtx*/nullptr); })
, PhysicalFinalizingTransformer([] () { return CreateDqsFinalizingOptTransformer(); })
, TypeAnnotationTransformer([state] () { return CreateDqsDataSinkTypeAnnotationTransformer(state->TypeCtx); })
diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp
index 7b85f56f93f..e26956545e9 100644
--- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp
+++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.cpp
@@ -1,134 +1,134 @@
-#include "dq_pq_read_actor.h"
-#include "probes.h"
-
+#include "dq_pq_read_actor.h"
+#include "probes.h"
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h>
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/dq/common/dq_common.h>
#include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h>
#include <ydb/library/yql/minikql/mkql_alloc.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/providers/pq/proto/dq_io_state.pb.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
#include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h>
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/event_local.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-#include <library/cpp/lwtrace/mon/mon_lwtrace.h>
-
-#include <util/generic/algorithm.h>
-#include <util/generic/hash.h>
-#include <util/generic/utility.h>
-
-#include <queue>
-#include <variant>
-
-namespace NKikimrServices {
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/event_local.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+#include <library/cpp/lwtrace/mon/mon_lwtrace.h>
+
+#include <util/generic/algorithm.h>
+#include <util/generic/hash.h>
+#include <util/generic/utility.h>
+
+#include <queue>
+#include <variant>
+
+namespace NKikimrServices {
// using constant value from ydb/core/protos/services.proto
// but to avoid peerdir on ydb/core/protos we introduce this constant
- constexpr ui32 KQP_COMPUTE = 535;
-};
-
-const TString LogPrefix = "PQ sink. ";
-
-#define SINK_LOG_T(s) \
- LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_D(s) \
- LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_I(s) \
- LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_W(s) \
- LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_N(s) \
- LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_E(s) \
- LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_C(s) \
- LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG(prio, s) \
- LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-
-namespace NYql::NDq {
-
-using namespace NActors;
-using namespace NLog;
-using namespace NKikimr::NMiniKQL;
-
-constexpr ui32 StateVersion = 1;
-
-namespace {
-
-LWTRACE_USING(DQ_PQ_PROVIDER);
-
-struct TEvPrivate {
- // Event ids
- enum EEv : ui32 {
- EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE),
-
- EvSourceDataReady = EvBegin,
-
- EvEnd
- };
-
- static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)");
-
- // Events
-
- struct TEvSourceDataReady : public TEventLocal<TEvSourceDataReady, EvSourceDataReady> {};
-};
-
-} // namespace
-
-class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqSourceActor {
-public:
- using TPartitionKey = std::pair<TString, ui64>; // Cluster, partition id.
-
- TDqPqReadActor(
- ui64 inputIndex,
- const TString& txId,
- const THolderFactory& holderFactory,
- NPq::NProto::TDqPqTopicSource&& sourceParams,
- NPq::NProto::TDqReadTaskParams&& readParams,
+ constexpr ui32 KQP_COMPUTE = 535;
+};
+
+const TString LogPrefix = "PQ sink. ";
+
+#define SINK_LOG_T(s) \
+ LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_D(s) \
+ LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_I(s) \
+ LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_W(s) \
+ LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_N(s) \
+ LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_E(s) \
+ LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_C(s) \
+ LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG(prio, s) \
+ LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+
+namespace NYql::NDq {
+
+using namespace NActors;
+using namespace NLog;
+using namespace NKikimr::NMiniKQL;
+
+constexpr ui32 StateVersion = 1;
+
+namespace {
+
+LWTRACE_USING(DQ_PQ_PROVIDER);
+
+struct TEvPrivate {
+ // Event ids
+ enum EEv : ui32 {
+ EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE),
+
+ EvSourceDataReady = EvBegin,
+
+ EvEnd
+ };
+
+ static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)");
+
+ // Events
+
+ struct TEvSourceDataReady : public TEventLocal<TEvSourceDataReady, EvSourceDataReady> {};
+};
+
+} // namespace
+
+class TDqPqReadActor : public NActors::TActor<TDqPqReadActor>, public IDqSourceActor {
+public:
+ using TPartitionKey = std::pair<TString, ui64>; // Cluster, partition id.
+
+ TDqPqReadActor(
+ ui64 inputIndex,
+ const TString& txId,
+ const THolderFactory& holderFactory,
+ NPq::NProto::TDqPqTopicSource&& sourceParams,
+ NPq::NProto::TDqReadTaskParams&& readParams,
NYdb::TDriver driver,
std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory,
- ICallbacks* callbacks,
- i64 bufferSize)
- : TActor<TDqPqReadActor>(&TDqPqReadActor::StateFunc)
- , InputIndex(inputIndex)
- , TxId(txId)
+ ICallbacks* callbacks,
+ i64 bufferSize)
+ : TActor<TDqPqReadActor>(&TDqPqReadActor::StateFunc)
+ , InputIndex(inputIndex)
+ , TxId(txId)
, BufferSize(bufferSize)
- , HolderFactory(holderFactory)
- , Driver(std::move(driver))
+ , HolderFactory(holderFactory)
+ , Driver(std::move(driver))
, CredentialsProviderFactory(std::move(credentialsProviderFactory))
- , SourceParams(std::move(sourceParams))
- , ReadParams(std::move(readParams))
+ , SourceParams(std::move(sourceParams))
+ , ReadParams(std::move(readParams))
, StartingMessageTimestamp(TInstant::Now())
- , Callbacks(callbacks)
- {
+ , Callbacks(callbacks)
+ {
Y_UNUSED(HolderFactory);
}
NYdb::NPersQueue::TPersQueueClientSettings GetPersQueueClientSettings() const {
- NYdb::NPersQueue::TPersQueueClientSettings opts;
- opts.Database(SourceParams.GetDatabase())
- .DiscoveryEndpoint(SourceParams.GetEndpoint())
- .EnableSsl(SourceParams.GetUseSsl())
+ NYdb::NPersQueue::TPersQueueClientSettings opts;
+ opts.Database(SourceParams.GetDatabase())
+ .DiscoveryEndpoint(SourceParams.GetEndpoint())
+ .EnableSsl(SourceParams.GetUseSsl())
.CredentialsProviderFactory(CredentialsProviderFactory);
-
+
return opts;
- }
-
+ }
+
static constexpr char ActorName[] = "DQ_PQ_READ_ACTOR";
-public:
+public:
void SaveState(const NDqProto::TCheckpoint& checkpoint, NDqProto::TSourceState& state) override {
NPq::NProto::TDqPqTopicSourceState stateProto;
@@ -138,15 +138,15 @@ public:
topic->SetDatabase(SourceParams.GetDatabase());
topic->SetTopicPath(SourceParams.GetTopicPath());
- for (const auto& [clusterAndPartition, offset] : PartitionToOffset) {
- const auto& [cluster, partition] = clusterAndPartition;
+ for (const auto& [clusterAndPartition, offset] : PartitionToOffset) {
+ const auto& [cluster, partition] = clusterAndPartition;
NPq::NProto::TDqPqTopicSourceState::TPartitionReadState* partitionState = stateProto.AddPartitions();
partitionState->SetTopicIndex(0); // Now we are supporting only one topic per source.
partitionState->SetCluster(cluster);
partitionState->SetPartition(partition);
partitionState->SetOffset(offset);
- }
-
+ }
+
stateProto.SetStartingMessageTimestampMs(StartingMessageTimestamp.MilliSeconds());
TString stateBlob;
@@ -156,10 +156,10 @@ public:
data->SetVersion(StateVersion);
data->SetBlob(stateBlob);
- DeferredCommits.emplace(checkpoint.GetId(), std::move(CurrentDeferredCommit));
- CurrentDeferredCommit = NYdb::NPersQueue::TDeferredCommit();
- }
-
+ DeferredCommits.emplace(checkpoint.GetId(), std::move(CurrentDeferredCommit));
+ CurrentDeferredCommit = NYdb::NPersQueue::TDeferredCommit();
+ }
+
void LoadState(const NDqProto::TSourceState& state) override {
TInstant minStartingMessageTs = state.DataSize() ? TInstant::Max() : StartingMessageTimestamp;
for (const auto& stateData : state.GetData()) {
@@ -187,20 +187,20 @@ public:
ReadSession.reset();
GetReadSession();
}
- }
-
+ }
+
void CommitState(const NDqProto::TCheckpoint& checkpoint) override {
- const auto checkpointId = checkpoint.GetId();
- while (!DeferredCommits.empty() && DeferredCommits.front().first <= checkpointId) {
- DeferredCommits.front().second.Commit();
- DeferredCommits.pop();
- }
- }
-
+ const auto checkpointId = checkpoint.GetId();
+ while (!DeferredCommits.empty() && DeferredCommits.front().first <= checkpointId) {
+ DeferredCommits.front().second.Commit();
+ DeferredCommits.pop();
+ }
+ }
+
ui64 GetInputIndex() const override {
- return InputIndex;
- };
-
+ return InputIndex;
+ };
+
NYdb::NPersQueue::TPersQueueClient& GetPersQueueClient() {
if (!PersQueueClient) {
PersQueueClient = std::make_unique<NYdb::NPersQueue::TPersQueueClient>(Driver, GetPersQueueClientSettings());
@@ -215,84 +215,84 @@ public:
return *ReadSession;
}
-private:
- STRICT_STFUNC(StateFunc,
- hFunc(TEvPrivate::TEvSourceDataReady, Handle);
- )
-
- void Handle(TEvPrivate::TEvSourceDataReady::TPtr& ev) {
+private:
+ STRICT_STFUNC(StateFunc,
+ hFunc(TEvPrivate::TEvSourceDataReady, Handle);
+ )
+
+ void Handle(TEvPrivate::TEvSourceDataReady::TPtr& ev) {
SubscribedOnEvent = false;
- Y_UNUSED(ev);
- Callbacks->OnNewSourceDataArrived(InputIndex);
- }
-
+ Y_UNUSED(ev);
+ Callbacks->OnNewSourceDataArrived(InputIndex);
+ }
+
// IActor & IDqSourceActor
void PassAway() override { // Is called from Compute Actor
if (ReadSession) {
ReadSession->Close(TDuration::Zero());
ReadSession.reset();
}
- PersQueueClient.reset();
+ PersQueueClient.reset();
TActor<TDqPqReadActor>::PassAway();
- }
-
+ }
+
i64 GetSourceData(NKikimr::NMiniKQL::TUnboxedValueVector& buffer, bool&, i64 freeSpace) override {
auto events = GetReadSession().GetEvents(false, TMaybe<size_t>(), static_cast<size_t>(Max<i64>(freeSpace, 0)));
-
- ui32 batchSize = 0;
- for (auto& event : events) {
- if (const auto* val = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent>(&event)) {
- batchSize += val->GetMessages().size();
- }
- }
- buffer.clear();
- buffer.reserve(batchSize);
-
- i64 usedSpace = 0;
- for (auto& event : events) {
- std::visit(TPQEventProcessor{*this, buffer, usedSpace}, event);
- }
-
- SubscribeOnNextEvent();
-
- return usedSpace;
- }
-
-private:
+
+ ui32 batchSize = 0;
+ for (auto& event : events) {
+ if (const auto* val = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent>(&event)) {
+ batchSize += val->GetMessages().size();
+ }
+ }
+ buffer.clear();
+ buffer.reserve(batchSize);
+
+ i64 usedSpace = 0;
+ for (auto& event : events) {
+ std::visit(TPQEventProcessor{*this, buffer, usedSpace}, event);
+ }
+
+ SubscribeOnNextEvent();
+
+ return usedSpace;
+ }
+
+private:
NYdb::NPersQueue::TReadSessionSettings GetReadSessionSettings() const {
- NYdb::NPersQueue::TTopicReadSettings topicReadSettings;
- topicReadSettings.Path(SourceParams.GetTopicPath());
- ui64 currentPartition = ReadParams.GetPartitioningParams().GetEachTopicPartitionGroupId();
- do {
- topicReadSettings.AppendPartitionGroupIds(currentPartition + 1); // 1-based.
- currentPartition += ReadParams.GetPartitioningParams().GetDqPartitionsCount();
- } while (currentPartition < ReadParams.GetPartitioningParams().GetTopicPartitionsCount());
-
- return NYdb::NPersQueue::TReadSessionSettings()
- .DisableClusterDiscovery(SourceParams.GetClusterType() == NPq::NProto::DataStreams)
- .AppendTopics(topicReadSettings)
- .ConsumerName(SourceParams.GetConsumerName())
+ NYdb::NPersQueue::TTopicReadSettings topicReadSettings;
+ topicReadSettings.Path(SourceParams.GetTopicPath());
+ ui64 currentPartition = ReadParams.GetPartitioningParams().GetEachTopicPartitionGroupId();
+ do {
+ topicReadSettings.AppendPartitionGroupIds(currentPartition + 1); // 1-based.
+ currentPartition += ReadParams.GetPartitioningParams().GetDqPartitionsCount();
+ } while (currentPartition < ReadParams.GetPartitioningParams().GetTopicPartitionsCount());
+
+ return NYdb::NPersQueue::TReadSessionSettings()
+ .DisableClusterDiscovery(SourceParams.GetClusterType() == NPq::NProto::DataStreams)
+ .AppendTopics(topicReadSettings)
+ .ConsumerName(SourceParams.GetConsumerName())
.MaxMemoryUsageBytes(BufferSize)
.StartingMessageTimestamp(StartingMessageTimestamp);
- }
-
- void UpdateStateWithNewReadData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& event) {
- if (event.GetMessages().empty()) {
- return;
- }
-
- assert(MaxElementBy(event.GetMessages(), [](const auto& message){ return message.GetOffset(); })
- ->GetOffset() == event.GetMessages().back().GetOffset());
-
- const auto maxOffset = event.GetMessages().back().GetOffset();
- PartitionToOffset[MakePartitionKey(event.GetPartitionStream())] = maxOffset + 1; // Next offset to read from.
- }
-
- static TPartitionKey MakePartitionKey(const NYdb::NPersQueue::TPartitionStream::TPtr& partitionStreamPtr) {
- return std::make_pair(partitionStreamPtr->GetCluster(), partitionStreamPtr->GetPartitionId());
- }
-
- void SubscribeOnNextEvent() {
+ }
+
+ void UpdateStateWithNewReadData(const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& event) {
+ if (event.GetMessages().empty()) {
+ return;
+ }
+
+ assert(MaxElementBy(event.GetMessages(), [](const auto& message){ return message.GetOffset(); })
+ ->GetOffset() == event.GetMessages().back().GetOffset());
+
+ const auto maxOffset = event.GetMessages().back().GetOffset();
+ PartitionToOffset[MakePartitionKey(event.GetPartitionStream())] = maxOffset + 1; // Next offset to read from.
+ }
+
+ static TPartitionKey MakePartitionKey(const NYdb::NPersQueue::TPartitionStream::TPtr& partitionStreamPtr) {
+ return std::make_pair(partitionStreamPtr->GetCluster(), partitionStreamPtr->GetPartitionId());
+ }
+
+ void SubscribeOnNextEvent() {
if (!SubscribedOnEvent) {
SubscribedOnEvent = true;
NActors::TActorSystem* actorSystem = NActors::TActivationContext::ActorSystem();
@@ -300,129 +300,129 @@ private:
actorSystem->Send(selfId, new TEvPrivate::TEvSourceDataReady());
});
}
- }
-
- struct TPQEventProcessor {
- void operator()(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& event) {
- for (const auto& message : event.GetMessages()) {
- const TString& data = message.GetData();
-
- LWPROBE(PqReadDataReceived, Self.TxId, Self.SourceParams.GetTopicPath(), data);
- SINK_LOG_T("Data received: " << data);
-
+ }
+
+ struct TPQEventProcessor {
+ void operator()(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& event) {
+ for (const auto& message : event.GetMessages()) {
+ const TString& data = message.GetData();
+
+ LWPROBE(PqReadDataReceived, Self.TxId, Self.SourceParams.GetTopicPath(), data);
+ SINK_LOG_T("Data received: " << data);
+
Batch.emplace_back(NKikimr::NMiniKQL::MakeString(NUdf::TStringRef(data.Data(), data.Size())));
UsedSpace += data.Size();
- }
- Self.UpdateStateWithNewReadData(event);
- Self.CurrentDeferredCommit.Add(event);
- }
-
- void operator()(NYdb::NPersQueue::TSessionClosedEvent& ev) {
- ythrow yexception() << "Read session to topic \"" << Self.SourceParams.GetTopicPath()
- << "\" was closed: " << ev.DebugString();
- }
-
- void operator()(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&) { }
-
- void operator()(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent& event) {
- TMaybe<ui64> readOffset;
- const auto offsetIt = Self.PartitionToOffset.find(MakePartitionKey(event.GetPartitionStream()));
- if (offsetIt != Self.PartitionToOffset.end()) {
- readOffset = offsetIt->second;
- }
- event.Confirm(readOffset);
- }
-
- void operator()(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent& event) {
- event.Confirm();
- }
-
- void operator()(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&) { }
-
- void operator()(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&) { }
-
- TDqPqReadActor& Self;
- TUnboxedValueVector& Batch;
- i64& UsedSpace;
- };
-
-private:
- const ui64 InputIndex;
- const TString TxId;
+ }
+ Self.UpdateStateWithNewReadData(event);
+ Self.CurrentDeferredCommit.Add(event);
+ }
+
+ void operator()(NYdb::NPersQueue::TSessionClosedEvent& ev) {
+ ythrow yexception() << "Read session to topic \"" << Self.SourceParams.GetTopicPath()
+ << "\" was closed: " << ev.DebugString();
+ }
+
+ void operator()(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&) { }
+
+ void operator()(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent& event) {
+ TMaybe<ui64> readOffset;
+ const auto offsetIt = Self.PartitionToOffset.find(MakePartitionKey(event.GetPartitionStream()));
+ if (offsetIt != Self.PartitionToOffset.end()) {
+ readOffset = offsetIt->second;
+ }
+ event.Confirm(readOffset);
+ }
+
+ void operator()(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent& event) {
+ event.Confirm();
+ }
+
+ void operator()(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&) { }
+
+ void operator()(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&) { }
+
+ TDqPqReadActor& Self;
+ TUnboxedValueVector& Batch;
+ i64& UsedSpace;
+ };
+
+private:
+ const ui64 InputIndex;
+ const TString TxId;
const i64 BufferSize;
- const THolderFactory& HolderFactory;
+ const THolderFactory& HolderFactory;
NYdb::TDriver Driver;
std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactory;
- const NPq::NProto::TDqPqTopicSource SourceParams;
- const NPq::NProto::TDqReadTaskParams ReadParams;
- std::unique_ptr<NYdb::NPersQueue::TPersQueueClient> PersQueueClient;
- std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession;
- NThreading::TFuture<void> EventFuture;
- THashMap<TPartitionKey, ui64> PartitionToOffset; // {cluster, partition} -> offset of next event.
+ const NPq::NProto::TDqPqTopicSource SourceParams;
+ const NPq::NProto::TDqReadTaskParams ReadParams;
+ std::unique_ptr<NYdb::NPersQueue::TPersQueueClient> PersQueueClient;
+ std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession;
+ NThreading::TFuture<void> EventFuture;
+ THashMap<TPartitionKey, ui64> PartitionToOffset; // {cluster, partition} -> offset of next event.
TInstant StartingMessageTimestamp;
ICallbacks* const Callbacks;
- std::queue<std::pair<ui64, NYdb::NPersQueue::TDeferredCommit>> DeferredCommits;
- NYdb::NPersQueue::TDeferredCommit CurrentDeferredCommit;
+ std::queue<std::pair<ui64, NYdb::NPersQueue::TDeferredCommit>> DeferredCommits;
+ NYdb::NPersQueue::TDeferredCommit CurrentDeferredCommit;
bool SubscribedOnEvent = false;
-};
-
+};
+
std::pair<IDqSourceActor*, NActors::IActor*> CreateDqPqReadActor(
NPq::NProto::TDqPqTopicSource&& settings,
- ui64 inputIndex,
- TTxId txId,
- const THashMap<TString, TString>& secureParams,
- const THashMap<TString, TString>& taskParams,
+ ui64 inputIndex,
+ TTxId txId,
+ const THashMap<TString, TString>& secureParams,
+ const THashMap<TString, TString>& taskParams,
NYdb::TDriver driver,
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
IDqSourceActor::ICallbacks* callbacks,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory,
i64 bufferSize
)
-{
- auto taskParamsIt = taskParams.find("pq");
- YQL_ENSURE(taskParamsIt != taskParams.end(), "Failed to get pq task params");
-
+{
+ auto taskParamsIt = taskParams.find("pq");
+ YQL_ENSURE(taskParamsIt != taskParams.end(), "Failed to get pq task params");
+
NPq::NProto::TDqReadTaskParams readTaskParamsMsg;
- YQL_ENSURE(readTaskParamsMsg.ParseFromString(taskParamsIt->second), "Failed to parse DqPqRead task params");
-
+ YQL_ENSURE(readTaskParamsMsg.ParseFromString(taskParamsIt->second), "Failed to parse DqPqRead task params");
+
const TString& tokenName = settings.GetToken().GetName();
const TString token = secureParams.Value(tokenName, TString());
- const bool addBearerToToken = settings.GetAddBearerToToken();
+ const bool addBearerToToken = settings.GetAddBearerToToken();
- TDqPqReadActor* actor = new TDqPqReadActor(
- inputIndex,
- std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId),
+ TDqPqReadActor* actor = new TDqPqReadActor(
+ inputIndex,
+ std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId),
holderFactory,
- std::move(settings),
- std::move(readTaskParamsMsg),
+ std::move(settings),
+ std::move(readTaskParamsMsg),
std::move(driver),
- CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken),
+ CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken),
callbacks,
bufferSize
);
- return {actor, actor};
-}
-
+ return {actor, actor};
+}
+
void RegisterDqPqReadActorFactory(TDqSourceFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) {
factory.Register<NPq::NProto::TDqPqTopicSource>("PqSource",
- [driver = std::move(driver), credentialsFactory = std::move(credentialsFactory)](
- NPq::NProto::TDqPqTopicSource&& settings,
- IDqSourceActorFactory::TArguments&& args)
- {
- NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(DQ_PQ_PROVIDER));
- return CreateDqPqReadActor(
- std::move(settings),
- args.InputIndex,
- args.TxId,
- args.SecureParams,
- args.TaskParams,
- driver,
- credentialsFactory,
- args.Callback,
- args.HolderFactory);
+ [driver = std::move(driver), credentialsFactory = std::move(credentialsFactory)](
+ NPq::NProto::TDqPqTopicSource&& settings,
+ IDqSourceActorFactory::TArguments&& args)
+ {
+ NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(DQ_PQ_PROVIDER));
+ return CreateDqPqReadActor(
+ std::move(settings),
+ args.InputIndex,
+ args.TxId,
+ args.SecureParams,
+ args.TaskParams,
+ driver,
+ credentialsFactory,
+ args.Callback,
+ args.HolderFactory);
});
}
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h
index 06999d9efe2..d7e2c82dd04 100644
--- a/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h
+++ b/ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h
@@ -1,5 +1,5 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h>
@@ -8,33 +8,33 @@
#include <ydb/library/yql/providers/pq/proto/dq_io.pb.h>
#include <ydb/library/yql/providers/pq/proto/dq_task_params.pb.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
-
-#include <library/cpp/actors/core/actor.h>
-
-#include <util/generic/size_literals.h>
-#include <util/system/types.h>
-
-
-namespace NYql::NDq {
+
+#include <library/cpp/actors/core/actor.h>
+
+#include <util/generic/size_literals.h>
+#include <util/system/types.h>
+
+
+namespace NYql::NDq {
class TDqSourceFactory;
-
-const i64 PQReadDefaultFreeSpace = 16_MB;
-
+
+const i64 PQReadDefaultFreeSpace = 16_MB;
+
std::pair<IDqSourceActor*, NActors::IActor*> CreateDqPqReadActor(
NPq::NProto::TDqPqTopicSource&& settings,
- ui64 inputIndex,
- TTxId txId,
- const THashMap<TString, TString>& secureParams,
- const THashMap<TString, TString>& taskParams,
+ ui64 inputIndex,
+ TTxId txId,
+ const THashMap<TString, TString>& secureParams,
+ const THashMap<TString, TString>& taskParams,
NYdb::TDriver driver,
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
IDqSourceActor::ICallbacks* callback,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory,
i64 bufferSize = PQReadDefaultFreeSpace
);
-
+
void RegisterDqPqReadActorFactory(TDqSourceFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory);
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp
index 5c8242229a4..85fbc91c01d 100644
--- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp
+++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.cpp
@@ -1,177 +1,177 @@
-#include "dq_pq_write_actor.h"
-#include "probes.h"
-
+#include "dq_pq_write_actor.h"
+#include "probes.h"
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h>
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/dq/common/dq_common.h>
#include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h>
#include <ydb/library/yql/minikql/mkql_alloc.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/providers/pq/proto/dq_io_state.pb.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
#include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h>
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/event_local.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-#include <library/cpp/lwtrace/mon/mon_lwtrace.h>
-
-#include <util/generic/algorithm.h>
-#include <util/generic/hash.h>
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/event_local.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+#include <library/cpp/lwtrace/mon/mon_lwtrace.h>
+
+#include <util/generic/algorithm.h>
+#include <util/generic/hash.h>
#include <util/string/builder.h>
-
-#include <algorithm>
-#include <queue>
-#include <variant>
-
-namespace NKikimrServices {
+
+#include <algorithm>
+#include <queue>
+#include <variant>
+
+namespace NKikimrServices {
// using constant value from ydb/core/protos/services.proto
// but to avoid peerdir on ydb/core/protos we introduce this constant
- constexpr ui32 KQP_COMPUTE = 535;
-};
-
-const TString LogPrefix = "PQ sink. ";
-
-#define SINK_LOG_T(s) \
- LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_D(s) \
- LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_I(s) \
- LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_W(s) \
- LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_N(s) \
- LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_E(s) \
- LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG_C(s) \
- LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-#define SINK_LOG(prio, s) \
- LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
-
-namespace NYql::NDq {
-
-using namespace NActors;
-using namespace NLog;
-using namespace NKikimr::NMiniKQL;
-
-constexpr ui32 StateVersion = 1;
-constexpr ui32 MaxMessageSize = 1_MB;
-
-namespace {
-
-LWTRACE_USING(DQ_PQ_PROVIDER);
-
-struct TEvPrivate {
- // Event ids
- enum EEv : ui32 {
- EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE),
-
- EvPqEventsReady = EvBegin,
-
- EvEnd
- };
-
- static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)");
-
- // Events
-
- struct TEvPqEventsReady : public TEventLocal<TEvPqEventsReady, EvPqEventsReady> {};
-};
-
-} // namespace
-
-class TDqPqWriteActor : public NActors::TActor<TDqPqWriteActor>, public IDqSinkActor {
-public:
- TDqPqWriteActor(
- ui64 outputIndex,
- const TString& txId,
- NPq::NProto::TDqPqTopicSink&& sinkParams,
+ constexpr ui32 KQP_COMPUTE = 535;
+};
+
+const TString LogPrefix = "PQ sink. ";
+
+#define SINK_LOG_T(s) \
+ LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_D(s) \
+ LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_I(s) \
+ LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_W(s) \
+ LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_N(s) \
+ LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_E(s) \
+ LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG_C(s) \
+ LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+#define SINK_LOG(prio, s) \
+ LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, LogPrefix << s)
+
+namespace NYql::NDq {
+
+using namespace NActors;
+using namespace NLog;
+using namespace NKikimr::NMiniKQL;
+
+constexpr ui32 StateVersion = 1;
+constexpr ui32 MaxMessageSize = 1_MB;
+
+namespace {
+
+LWTRACE_USING(DQ_PQ_PROVIDER);
+
+struct TEvPrivate {
+ // Event ids
+ enum EEv : ui32 {
+ EvBegin = EventSpaceBegin(NActors::TEvents::ES_PRIVATE),
+
+ EvPqEventsReady = EvBegin,
+
+ EvEnd
+ };
+
+ static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE)");
+
+ // Events
+
+ struct TEvPqEventsReady : public TEventLocal<TEvPqEventsReady, EvPqEventsReady> {};
+};
+
+} // namespace
+
+class TDqPqWriteActor : public NActors::TActor<TDqPqWriteActor>, public IDqSinkActor {
+public:
+ TDqPqWriteActor(
+ ui64 outputIndex,
+ const TString& txId,
+ NPq::NProto::TDqPqTopicSink&& sinkParams,
NYdb::TDriver driver,
std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory,
IDqSinkActor::ICallbacks* callbacks,
- i64 freeSpace)
- : TActor<TDqPqWriteActor>(&TDqPqWriteActor::StateFunc)
- , OutputIndex(outputIndex)
- , TxId(txId)
- , SinkParams(std::move(sinkParams))
+ i64 freeSpace)
+ : TActor<TDqPqWriteActor>(&TDqPqWriteActor::StateFunc)
+ , OutputIndex(outputIndex)
+ , TxId(txId)
+ , SinkParams(std::move(sinkParams))
, Driver(std::move(driver))
, CredentialsProviderFactory(credentialsProviderFactory)
- , Callbacks(callbacks)
- , FreeSpace(freeSpace)
+ , Callbacks(callbacks)
+ , FreeSpace(freeSpace)
, PersQueueClient(Driver, GetPersQueueClientSettings())
- { }
-
+ { }
+
static constexpr char ActorName[] = "DQ_PQ_WRITE_ACTOR";
-public:
+public:
void SendData(
- NKikimr::NMiniKQL::TUnboxedValueVector&& batch,
+ NKikimr::NMiniKQL::TUnboxedValueVector&& batch,
i64 dataSize,
const TMaybe<NDqProto::TCheckpoint>& checkpoint,
bool finished) override
- {
+ {
Y_UNUSED(finished);
- Y_UNUSED(dataSize);
-
- CreateSessionIfNotExists();
-
+ Y_UNUSED(dataSize);
+
+ CreateSessionIfNotExists();
+
for (const NUdf::TUnboxedValue& item : batch) {
- if (!item.IsBoxed()) {
- Fail("Struct with single field was expected");
- return;
- }
-
+ if (!item.IsBoxed()) {
+ Fail("Struct with single field was expected");
+ return;
+ }
+
const NUdf::TUnboxedValue dataCol = item.GetElement(0);
-
- if (!dataCol.IsString() && !dataCol.IsEmbedded()) {
- Fail(TStringBuilder() << "Non string value could not be written to YDS stream");
- return;
- }
-
+
+ if (!dataCol.IsString() && !dataCol.IsEmbedded()) {
+ Fail(TStringBuilder() << "Non string value could not be written to YDS stream");
+ return;
+ }
+
TString data(dataCol.AsStringRef());
-
- LWPROBE(PqWriteDataToSend, TxId, SinkParams.GetTopicPath(), data);
- SINK_LOG_T("Received data for sending: " << data);
-
- const auto messageSize = GetItemSize(data);
- if (messageSize > MaxMessageSize) {
- Fail(TStringBuilder() << "Max message size for YDS is " << MaxMessageSize
- << " bytes but received message with size of " << messageSize << " bytes");
- return;
- }
-
- FreeSpace -= messageSize;
- Buffer.push(std::move(data));
- }
-
- if (checkpoint) {
- if (Buffer.empty()) {
+
+ LWPROBE(PqWriteDataToSend, TxId, SinkParams.GetTopicPath(), data);
+ SINK_LOG_T("Received data for sending: " << data);
+
+ const auto messageSize = GetItemSize(data);
+ if (messageSize > MaxMessageSize) {
+ Fail(TStringBuilder() << "Max message size for YDS is " << MaxMessageSize
+ << " bytes but received message with size of " << messageSize << " bytes");
+ return;
+ }
+
+ FreeSpace -= messageSize;
+ Buffer.push(std::move(data));
+ }
+
+ if (checkpoint) {
+ if (Buffer.empty()) {
Callbacks->OnSinkStateSaved(BuildState(), OutputIndex, *checkpoint);
- } else {
- DeferredCheckpoints.emplace(NextSeqNo + Buffer.size() - 1, *checkpoint);
- }
- }
-
- if (!Buffer.empty() && ContinuationToken) {
- WriteNextMessage(std::move(*ContinuationToken));
- ContinuationToken = std::nullopt;
- }
-
- while (HandleNewPQEvents()) { } // Write messages while new continuationTokens are arriving
-
- if (FreeSpace <= 0) {
- ShouldNotifyNewFreeSpace = true;
- }
- };
-
+ } else {
+ DeferredCheckpoints.emplace(NextSeqNo + Buffer.size() - 1, *checkpoint);
+ }
+ }
+
+ if (!Buffer.empty() && ContinuationToken) {
+ WriteNextMessage(std::move(*ContinuationToken));
+ ContinuationToken = std::nullopt;
+ }
+
+ while (HandleNewPQEvents()) { } // Write messages while new continuationTokens are arriving
+
+ if (FreeSpace <= 0) {
+ ShouldNotifyNewFreeSpace = true;
+ }
+ };
+
void LoadState(const NDqProto::TSinkState& state) override {
Y_VERIFY(NextSeqNo == 1);
const auto& data = state.GetData().GetStateData();
@@ -180,112 +180,112 @@ public:
YQL_ENSURE(stateProto.ParseFromString(data.GetBlob()), "Serialized state is corrupted");
SourceId = stateProto.GetSourceId();
ConfirmedSeqNo = stateProto.GetConfirmedSeqNo();
- NextSeqNo = ConfirmedSeqNo + 1;
+ NextSeqNo = ConfirmedSeqNo + 1;
return;
- }
+ }
ythrow yexception() << "Invalid state version " << data.GetVersion();
- }
-
- void CommitState(const NDqProto::TCheckpoint& checkpoint) override {
- Y_UNUSED(checkpoint);
- }
-
- i64 GetFreeSpace() const override {
- return FreeSpace;
- };
-
- ui64 GetOutputIndex() const override {
- return OutputIndex;
- };
-
-private:
- STRICT_STFUNC(StateFunc,
- hFunc(TEvPrivate::TEvPqEventsReady, Handle);
- )
-
- void Handle(TEvPrivate::TEvPqEventsReady::TPtr&) {
- while (HandleNewPQEvents()) { }
- SubscribeOnNextEvent();
- }
-
+ }
+
+ void CommitState(const NDqProto::TCheckpoint& checkpoint) override {
+ Y_UNUSED(checkpoint);
+ }
+
+ i64 GetFreeSpace() const override {
+ return FreeSpace;
+ };
+
+ ui64 GetOutputIndex() const override {
+ return OutputIndex;
+ };
+
+private:
+ STRICT_STFUNC(StateFunc,
+ hFunc(TEvPrivate::TEvPqEventsReady, Handle);
+ )
+
+ void Handle(TEvPrivate::TEvPqEventsReady::TPtr&) {
+ while (HandleNewPQEvents()) { }
+ SubscribeOnNextEvent();
+ }
+
// IActor & IDqSinkActor
void PassAway() override { // Is called from Compute Actor
- if (WriteSession) {
- WriteSession->Close(TDuration::Zero());
- }
+ if (WriteSession) {
+ WriteSession->Close(TDuration::Zero());
+ }
TActor<TDqPqWriteActor>::PassAway();
- }
-
-private:
- const TString& GetSourceId() {
- if (!SourceId) {
- SourceId = CreateGuidAsString(); // Not loaded from state, so this is the first run.
- }
- return SourceId;
- }
-
- NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() {
- return NYdb::NPersQueue::TWriteSessionSettings(SinkParams.GetTopicPath(), GetSourceId())
- .MaxMemoryUsage(FreeSpace)
- .ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Auto)
- .Codec(SinkParams.GetClusterType() == NPq::NProto::DataStreams
- ? NYdb::NPersQueue::ECodec::RAW
- : NYdb::NPersQueue::ECodec::GZIP);
- }
-
+ }
+
+private:
+ const TString& GetSourceId() {
+ if (!SourceId) {
+ SourceId = CreateGuidAsString(); // Not loaded from state, so this is the first run.
+ }
+ return SourceId;
+ }
+
+ NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() {
+ return NYdb::NPersQueue::TWriteSessionSettings(SinkParams.GetTopicPath(), GetSourceId())
+ .MaxMemoryUsage(FreeSpace)
+ .ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Auto)
+ .Codec(SinkParams.GetClusterType() == NPq::NProto::DataStreams
+ ? NYdb::NPersQueue::ECodec::RAW
+ : NYdb::NPersQueue::ECodec::GZIP);
+ }
+
NYdb::NPersQueue::TPersQueueClientSettings GetPersQueueClientSettings() {
return NYdb::NPersQueue::TPersQueueClientSettings()
- .Database(SinkParams.GetDatabase())
- .DiscoveryEndpoint(SinkParams.GetEndpoint())
- .EnableSsl(SinkParams.GetUseSsl())
+ .Database(SinkParams.GetDatabase())
+ .DiscoveryEndpoint(SinkParams.GetEndpoint())
+ .EnableSsl(SinkParams.GetUseSsl())
.CredentialsProviderFactory(CredentialsProviderFactory);
- }
-
- static i64 GetItemSize(const TString& item) {
- return std::max(static_cast<i64>(item.size()), static_cast<i64>(1));
- }
-
- void CreateSessionIfNotExists() {
- if (!WriteSession) {
- WriteSession = PersQueueClient.CreateWriteSession(GetWriteSessionSettings());
- SubscribeOnNextEvent();
- }
- }
-
- void SubscribeOnNextEvent() {
- if (!WriteSession) {
- return;
- }
-
- NActors::TActorSystem* actorSystem = NActors::TActivationContext::ActorSystem();
- EventFuture = WriteSession->WaitEvent().Subscribe([actorSystem, selfId = SelfId()](const auto&){
- actorSystem->Send(selfId, new TEvPrivate::TEvPqEventsReady());
- });
- }
-
- bool HandleNewPQEvents() {
- if (!WriteSession) {
- return false;
- }
-
- auto events = WriteSession->GetEvents();
- for (auto& event : events) {
+ }
+
+ static i64 GetItemSize(const TString& item) {
+ return std::max(static_cast<i64>(item.size()), static_cast<i64>(1));
+ }
+
+ void CreateSessionIfNotExists() {
+ if (!WriteSession) {
+ WriteSession = PersQueueClient.CreateWriteSession(GetWriteSessionSettings());
+ SubscribeOnNextEvent();
+ }
+ }
+
+ void SubscribeOnNextEvent() {
+ if (!WriteSession) {
+ return;
+ }
+
+ NActors::TActorSystem* actorSystem = NActors::TActivationContext::ActorSystem();
+ EventFuture = WriteSession->WaitEvent().Subscribe([actorSystem, selfId = SelfId()](const auto&){
+ actorSystem->Send(selfId, new TEvPrivate::TEvPqEventsReady());
+ });
+ }
+
+ bool HandleNewPQEvents() {
+ if (!WriteSession) {
+ return false;
+ }
+
+ auto events = WriteSession->GetEvents();
+ for (auto& event : events) {
auto issues = std::visit(TPQEventProcessor{*this}, event);
if (issues) {
- WriteSession->Close(TDuration::Zero());
- WriteSession.reset();
- Callbacks->OnSinkError(OutputIndex, *issues, true);
- break;
- }
-
- if (ShouldNotifyNewFreeSpace && FreeSpace > 0) {
- Callbacks->ResumeExecution();
- ShouldNotifyNewFreeSpace = false;
- }
- }
- return !events.empty();
- }
-
+ WriteSession->Close(TDuration::Zero());
+ WriteSession.reset();
+ Callbacks->OnSinkError(OutputIndex, *issues, true);
+ break;
+ }
+
+ if (ShouldNotifyNewFreeSpace && FreeSpace > 0) {
+ Callbacks->ResumeExecution();
+ ShouldNotifyNewFreeSpace = false;
+ }
+ }
+ return !events.empty();
+ }
+
NDqProto::TSinkState BuildState() {
NPq::NProto::TDqPqTopicSinkState stateProto;
stateProto.SetSourceId(GetSourceId());
@@ -298,138 +298,138 @@ private:
data->SetVersion(StateVersion);
data->SetBlob(serializedState);
return sinkState;
- }
-
- void WriteNextMessage(NYdb::NPersQueue::TContinuationToken&& token) {
- WriteSession->Write(std::move(token), Buffer.front(), NextSeqNo++);
- WaitingAcks.push(GetItemSize(Buffer.front()));
- Buffer.pop();
- }
-
- void Fail(TString message) {
- TIssues issues;
- issues.AddIssue(message);
- Callbacks->OnSinkError(OutputIndex, issues, true);
- return;
- }
-
- struct TPQEventProcessor {
+ }
+
+ void WriteNextMessage(NYdb::NPersQueue::TContinuationToken&& token) {
+ WriteSession->Write(std::move(token), Buffer.front(), NextSeqNo++);
+ WaitingAcks.push(GetItemSize(Buffer.front()));
+ Buffer.pop();
+ }
+
+ void Fail(TString message) {
+ TIssues issues;
+ issues.AddIssue(message);
+ Callbacks->OnSinkError(OutputIndex, issues, true);
+ return;
+ }
+
+ struct TPQEventProcessor {
std::optional<TIssues> operator()(NYdb::NPersQueue::TSessionClosedEvent& ev) {
TIssues issues;
- issues.AddIssue(TStringBuilder() << "Write session to topic \"" << Self.SinkParams.GetTopicPath() << "\" was closed: " << ev.DebugString());
+ issues.AddIssue(TStringBuilder() << "Write session to topic \"" << Self.SinkParams.GetTopicPath() << "\" was closed: " << ev.DebugString());
return issues;
- }
-
+ }
+
std::optional<TIssues> operator()(NYdb::NPersQueue::TWriteSessionEvent::TAcksEvent& ev) {
- if (ev.Acks.empty()) {
- return std::nullopt;
- }
-
- //Y_VERIFY(Self.ConfirmedSeqNo == 0 || ev.Acks.front().SeqNo == Self.ConfirmedSeqNo + 1);
-
- for (auto it = ev.Acks.begin(); it != ev.Acks.end(); ++it) {
- //Y_VERIFY(it == ev.Acks.begin() || it->SeqNo == std::prev(it)->SeqNo + 1);
-
- if (it->State == NYdb::NPersQueue::TWriteSessionEvent::TWriteAck::EEventState::EES_DISCARDED) {
+ if (ev.Acks.empty()) {
+ return std::nullopt;
+ }
+
+ //Y_VERIFY(Self.ConfirmedSeqNo == 0 || ev.Acks.front().SeqNo == Self.ConfirmedSeqNo + 1);
+
+ for (auto it = ev.Acks.begin(); it != ev.Acks.end(); ++it) {
+ //Y_VERIFY(it == ev.Acks.begin() || it->SeqNo == std::prev(it)->SeqNo + 1);
+
+ if (it->State == NYdb::NPersQueue::TWriteSessionEvent::TWriteAck::EEventState::EES_DISCARDED) {
TIssues issues;
issues.AddIssue(TStringBuilder() << "Message with seqNo " << it->SeqNo << " was discarded");
return issues;
- }
-
- Self.FreeSpace += Self.WaitingAcks.front();
- Self.WaitingAcks.pop();
-
- if (!Self.DeferredCheckpoints.empty() && std::get<0>(Self.DeferredCheckpoints.front()) == it->SeqNo) {
- Self.ConfirmedSeqNo = it->SeqNo;
+ }
+
+ Self.FreeSpace += Self.WaitingAcks.front();
+ Self.WaitingAcks.pop();
+
+ if (!Self.DeferredCheckpoints.empty() && std::get<0>(Self.DeferredCheckpoints.front()) == it->SeqNo) {
+ Self.ConfirmedSeqNo = it->SeqNo;
Self.Callbacks->OnSinkStateSaved(Self.BuildState(), Self.OutputIndex, std::get<1>(Self.DeferredCheckpoints.front()));
- Self.DeferredCheckpoints.pop();
- }
- }
- Self.ConfirmedSeqNo = ev.Acks.back().SeqNo;
-
- return std::nullopt;
- }
-
+ Self.DeferredCheckpoints.pop();
+ }
+ }
+ Self.ConfirmedSeqNo = ev.Acks.back().SeqNo;
+
+ return std::nullopt;
+ }
+
std::optional<TIssues> operator()(NYdb::NPersQueue::TWriteSessionEvent::TReadyToAcceptEvent& ev) {
//Y_VERIFY(!Self.ContinuationToken);
-
- if (!Self.Buffer.empty()) {
- Self.WriteNextMessage(std::move(ev.ContinuationToken));
- return std::nullopt;
- }
-
- Self.ContinuationToken = std::move(ev.ContinuationToken);
- return std::nullopt;
- }
-
- TDqPqWriteActor& Self;
- };
-
-private:
- const ui64 OutputIndex;
- const TString TxId;
- const NPq::NProto::TDqPqTopicSink SinkParams;
+
+ if (!Self.Buffer.empty()) {
+ Self.WriteNextMessage(std::move(ev.ContinuationToken));
+ return std::nullopt;
+ }
+
+ Self.ContinuationToken = std::move(ev.ContinuationToken);
+ return std::nullopt;
+ }
+
+ TDqPqWriteActor& Self;
+ };
+
+private:
+ const ui64 OutputIndex;
+ const TString TxId;
+ const NPq::NProto::TDqPqTopicSink SinkParams;
NYdb::TDriver Driver;
std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactory;
IDqSinkActor::ICallbacks* const Callbacks;
- i64 FreeSpace = 0;
-
- NYdb::NPersQueue::TPersQueueClient PersQueueClient;
- std::shared_ptr<NYdb::NPersQueue::IWriteSession> WriteSession;
- TString SourceId;
- ui64 NextSeqNo = 1;
- ui64 ConfirmedSeqNo = 0;
- std::optional<NYdb::NPersQueue::TContinuationToken> ContinuationToken;
- NThreading::TFuture<void> EventFuture;
- std::queue<i64> InflightMessageSizes;
- bool ShouldNotifyNewFreeSpace = false;
- std::queue<TString> Buffer;
- std::queue<i64> WaitingAcks; // Size of items which are waiting for acks (used to update free space)
- std::queue<std::tuple<ui64, NDqProto::TCheckpoint>> DeferredCheckpoints;
-};
-
+ i64 FreeSpace = 0;
+
+ NYdb::NPersQueue::TPersQueueClient PersQueueClient;
+ std::shared_ptr<NYdb::NPersQueue::IWriteSession> WriteSession;
+ TString SourceId;
+ ui64 NextSeqNo = 1;
+ ui64 ConfirmedSeqNo = 0;
+ std::optional<NYdb::NPersQueue::TContinuationToken> ContinuationToken;
+ NThreading::TFuture<void> EventFuture;
+ std::queue<i64> InflightMessageSizes;
+ bool ShouldNotifyNewFreeSpace = false;
+ std::queue<TString> Buffer;
+ std::queue<i64> WaitingAcks; // Size of items which are waiting for acks (used to update free space)
+ std::queue<std::tuple<ui64, NDqProto::TCheckpoint>> DeferredCheckpoints;
+};
+
std::pair<IDqSinkActor*, NActors::IActor*> CreateDqPqWriteActor(
NPq::NProto::TDqPqTopicSink&& settings,
- ui64 outputIndex,
- TTxId txId,
- const THashMap<TString, TString>& secureParams,
+ ui64 outputIndex,
+ TTxId txId,
+ const THashMap<TString, TString>& secureParams,
NYdb::TDriver driver,
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
IDqSinkActor::ICallbacks* callbacks,
- i64 freeSpace)
-{
+ i64 freeSpace)
+{
const TString& tokenName = settings.GetToken().GetName();
const TString token = secureParams.Value(tokenName, TString());
- const bool addBearerToToken = settings.GetAddBearerToToken();
+ const bool addBearerToToken = settings.GetAddBearerToToken();
- TDqPqWriteActor* actor = new TDqPqWriteActor(
- outputIndex,
- std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId),
- std::move(settings),
+ TDqPqWriteActor* actor = new TDqPqWriteActor(
+ outputIndex,
+ std::holds_alternative<ui64>(txId) ? ToString(txId) : std::get<TString>(txId),
+ std::move(settings),
std::move(driver),
- CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken),
- callbacks,
- freeSpace);
- return {actor, actor};
-}
-
+ CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token, addBearerToToken),
+ callbacks,
+ freeSpace);
+ return {actor, actor};
+}
+
void RegisterDqPqWriteActorFactory(TDqSinkFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) {
factory.Register<NPq::NProto::TDqPqTopicSink>("PqSink",
[driver = std::move(driver), credentialsFactory = std::move(credentialsFactory)](
- NPq::NProto::TDqPqTopicSink&& settings,
- IDqSinkActorFactory::TArguments&& args)
- {
- NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(DQ_PQ_PROVIDER));
+ NPq::NProto::TDqPqTopicSink&& settings,
+ IDqSinkActorFactory::TArguments&& args)
+ {
+ NLwTraceMonPage::ProbeRegistry().AddProbesList(LWTRACE_GET_PROBES(DQ_PQ_PROVIDER));
return CreateDqPqWriteActor(
std::move(settings),
- args.OutputIndex,
- args.TxId,
- args.SecureParams,
+ args.OutputIndex,
+ args.TxId,
+ args.SecureParams,
driver,
credentialsFactory,
- args.Callback
+ args.Callback
);
- });
+ });
}
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h
index 90b393dfcc9..6736c8188b2 100644
--- a/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h
+++ b/ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h
@@ -1,5 +1,5 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h>
@@ -8,28 +8,28 @@
#include <ydb/library/yql/providers/pq/proto/dq_io.pb.h>
#include <ydb/library/yql/providers/pq/proto/dq_task_params.pb.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
-
-#include <library/cpp/actors/core/actor.h>
-
-#include <util/generic/size_literals.h>
-#include <util/system/types.h>
-
-namespace NYql::NDq {
-
-constexpr i64 DqPqDefaultFreeSpace = 16_MB;
-
+
+#include <library/cpp/actors/core/actor.h>
+
+#include <util/generic/size_literals.h>
+#include <util/system/types.h>
+
+namespace NYql::NDq {
+
+constexpr i64 DqPqDefaultFreeSpace = 16_MB;
+
std::pair<IDqSinkActor*, NActors::IActor*> CreateDqPqWriteActor(
NPq::NProto::TDqPqTopicSink&& settings,
- ui64 outputIndex,
- TTxId txId,
- const THashMap<TString, TString>& secureParams,
+ ui64 outputIndex,
+ TTxId txId,
+ const THashMap<TString, TString>& secureParams,
NYdb::TDriver driver,
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
IDqSinkActor::ICallbacks* callbacks,
- i64 freeSpace = DqPqDefaultFreeSpace);
-
+ i64 freeSpace = DqPqDefaultFreeSpace);
+
void RegisterDqPqWriteActorFactory(TDqSinkFactory& factory, NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory);
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/pq/async_io/probes.cpp b/ydb/library/yql/providers/pq/async_io/probes.cpp
index 5cafed75a5e..904821f4186 100644
--- a/ydb/library/yql/providers/pq/async_io/probes.cpp
+++ b/ydb/library/yql/providers/pq/async_io/probes.cpp
@@ -1,3 +1,3 @@
-#include "probes.h"
-
-LWTRACE_DEFINE_PROVIDER(DQ_PQ_PROVIDER)
+#include "probes.h"
+
+LWTRACE_DEFINE_PROVIDER(DQ_PQ_PROVIDER)
diff --git a/ydb/library/yql/providers/pq/async_io/probes.h b/ydb/library/yql/providers/pq/async_io/probes.h
index 1863894ca73..7bd876dc4d3 100644
--- a/ydb/library/yql/providers/pq/async_io/probes.h
+++ b/ydb/library/yql/providers/pq/async_io/probes.h
@@ -1,17 +1,17 @@
-#pragma once
-
-#include <library/cpp/lwtrace/all.h>
-
-#define DQ_PQ_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \
- PROBE(PqReadDataReceived, \
- GROUPS(), \
- TYPES(TString, TString, TString), \
- NAMES("queryId", "topic", "data")) \
- PROBE(PqWriteDataToSend, \
- GROUPS(), \
- TYPES(TString, TString, TString), \
- NAMES("queryId", "topic", "data")) \
-
-// DQ_PQ_PROVIDER
-
-LWTRACE_DECLARE_PROVIDER(DQ_PQ_PROVIDER)
+#pragma once
+
+#include <library/cpp/lwtrace/all.h>
+
+#define DQ_PQ_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \
+ PROBE(PqReadDataReceived, \
+ GROUPS(), \
+ TYPES(TString, TString, TString), \
+ NAMES("queryId", "topic", "data")) \
+ PROBE(PqWriteDataToSend, \
+ GROUPS(), \
+ TYPES(TString, TString, TString), \
+ NAMES("queryId", "topic", "data")) \
+
+// DQ_PQ_PROVIDER
+
+LWTRACE_DECLARE_PROVIDER(DQ_PQ_PROVIDER)
diff --git a/ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp b/ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp
index 7ddef8bda4a..ed6579022cf 100644
--- a/ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp
+++ b/ydb/library/yql/providers/pq/async_io/ut/dq_pq_read_actor_ut.cpp
@@ -1,169 +1,169 @@
-#include "ut_helpers.h"
-
+#include "ut_helpers.h"
+
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <thread>
-
-namespace NYql::NDq {
-
-Y_UNIT_TEST_SUITE(TDqPqReadActorTest) {
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <thread>
+
+namespace NYql::NDq {
+
+Y_UNIT_TEST_SUITE(TDqPqReadActorTest) {
Y_UNIT_TEST_F(TestReadFromTopic, TPqIoTestFixture) {
- const TString topicName = "ReadFromTopic";
+ const TString topicName = "ReadFromTopic";
InitSource(topicName);
-
- const std::vector<TString> data = { "1", "2", "3", "4" };
- PQWrite(data, topicName);
-
+
+ const std::vector<TString> data = { "1", "2", "3", "4" };
+ PQWrite(data, topicName);
+
auto result = SourceReadUntil<TString>(UVParser, 4);
- UNIT_ASSERT_EQUAL(result, data);
- }
-
+ UNIT_ASSERT_EQUAL(result, data);
+ }
+
Y_UNIT_TEST_F(ReadWithFreeSpace, TPqIoTestFixture) {
- const TString topicName = "ReadWithFreeSpace";
+ const TString topicName = "ReadWithFreeSpace";
InitSource(topicName);
-
- PQWrite({"data1", "data2", "data3"}, topicName);
-
- {
+
+ PQWrite({"data1", "data2", "data3"}, topicName);
+
+ {
auto result = SourceReadUntil<TString>(UVParser, 1, 1);
- std::vector<TString> expected {"data1"};
- UNIT_ASSERT_EQUAL(result, expected);
- }
-
+ std::vector<TString> expected {"data1"};
+ UNIT_ASSERT_EQUAL(result, expected);
+ }
+
UNIT_ASSERT_EQUAL(SourceRead<TString>(UVParser, 0).size(), 0);
UNIT_ASSERT_EQUAL(SourceRead<TString>(UVParser, -1).size(), 0);
- }
-
+ }
+
Y_UNIT_TEST_F(ReadNonExistentTopic, TPqIoTestFixture) {
- const TString topicName = "NonExistentTopic";
+ const TString topicName = "NonExistentTopic";
InitSource(topicName);
-
- while (true) {
- try {
+
+ while (true) {
+ try {
SourceRead<TString>(UVParser);
- } catch (yexception& e) {
- UNIT_ASSERT_STRING_CONTAINS(e.what(), "Read session to topic \"NonExistentTopic\" was closed");
- break;
- }
-
- sleep(1);
- }
- }
-
- Y_UNIT_TEST(TestSaveLoadPqRead) {
+ } catch (yexception& e) {
+ UNIT_ASSERT_STRING_CONTAINS(e.what(), "Read session to topic \"NonExistentTopic\" was closed");
+ break;
+ }
+
+ sleep(1);
+ }
+ }
+
+ Y_UNIT_TEST(TestSaveLoadPqRead) {
NDqProto::TSourceState state;
- const TString topicName = "SaveLoadPqRead";
-
- {
+ const TString topicName = "SaveLoadPqRead";
+
+ {
TPqIoTestFixture setup1;
setup1.InitSource(topicName);
-
- std::vector<TString> data {"data"};
- PQWrite(data, topicName);
-
- auto result = setup1.SourceReadUntil<TString>(UVParser, 1);
- UNIT_ASSERT_EQUAL(result, data);
-
- auto checkpoint = CreateCheckpoint();
+
+ std::vector<TString> data {"data"};
+ PQWrite(data, topicName);
+
+ auto result = setup1.SourceReadUntil<TString>(UVParser, 1);
+ UNIT_ASSERT_EQUAL(result, data);
+
+ auto checkpoint = CreateCheckpoint();
setup1.SaveSourceState(checkpoint, state);
- Cerr << "State saved" << Endl;
- }
-
+ Cerr << "State saved" << Endl;
+ }
+
NDqProto::TSourceState state2;
- {
+ {
TPqIoTestFixture setup2;
setup2.InitSource(topicName);
-
- std::vector<TString> data {"data"};
- PQWrite({"data"}, topicName);
-
- setup2.LoadSource(state);
- Cerr << "State loaded" << Endl;
- auto result = setup2.SourceReadUntil<TString>(UVParser, 1);
- UNIT_ASSERT_EQUAL(result, data);
-
- auto checkpoint = CreateCheckpoint();
+
+ std::vector<TString> data {"data"};
+ PQWrite({"data"}, topicName);
+
+ setup2.LoadSource(state);
+ Cerr << "State loaded" << Endl;
+ auto result = setup2.SourceReadUntil<TString>(UVParser, 1);
+ UNIT_ASSERT_EQUAL(result, data);
+
+ auto checkpoint = CreateCheckpoint();
setup2.SaveSourceState(checkpoint, state2);
-
- PQWrite({"futherData"}, topicName);
- }
-
+
+ PQWrite({"futherData"}, topicName);
+ }
+
NDqProto::TSourceState state3;
- {
+ {
TPqIoTestFixture setup3;
setup3.InitSource(topicName);
- setup3.LoadSource(state2);
-
- auto result = setup3.SourceReadUntil<TString>(UVParser, 1);
- std::vector<TString> data {"futherData"};
- UNIT_ASSERT_EQUAL(result, data);
-
- // pq session is steel alive
-
- PQWrite({"yetAnotherData"}, topicName);
-
- auto checkpoint = CreateCheckpoint();
+ setup3.LoadSource(state2);
+
+ auto result = setup3.SourceReadUntil<TString>(UVParser, 1);
+ std::vector<TString> data {"futherData"};
+ UNIT_ASSERT_EQUAL(result, data);
+
+ // pq session is steel alive
+
+ PQWrite({"yetAnotherData"}, topicName);
+
+ auto checkpoint = CreateCheckpoint();
setup3.SaveSourceState(checkpoint, state3);
- }
-
- // Load the first state and check it.
- {
+ }
+
+ // Load the first state and check it.
+ {
TPqIoTestFixture setup4;
setup4.InitSource(topicName);
- setup4.LoadSource(state);
-
- auto result = setup4.SourceReadUntil<TString>(UVParser, 3);
- std::vector<TString> data {"data", "futherData", "yetAnotherData"};
- UNIT_ASSERT_EQUAL(result, data);
- }
-
- // Load graphState2 and check it (offsets were saved).
- {
+ setup4.LoadSource(state);
+
+ auto result = setup4.SourceReadUntil<TString>(UVParser, 3);
+ std::vector<TString> data {"data", "futherData", "yetAnotherData"};
+ UNIT_ASSERT_EQUAL(result, data);
+ }
+
+ // Load graphState2 and check it (offsets were saved).
+ {
TPqIoTestFixture setup5;
setup5.InitSource(topicName);
- setup5.LoadSource(state2);
-
- auto result = setup5.SourceReadUntil<TString>(UVParser, 2);
- std::vector<TString> data {"futherData", "yetAnotherData"};
- UNIT_ASSERT_EQUAL(result, data);
- }
-
- // Load graphState3 and check it (other offsets).
- {
+ setup5.LoadSource(state2);
+
+ auto result = setup5.SourceReadUntil<TString>(UVParser, 2);
+ std::vector<TString> data {"futherData", "yetAnotherData"};
+ UNIT_ASSERT_EQUAL(result, data);
+ }
+
+ // Load graphState3 and check it (other offsets).
+ {
TPqIoTestFixture setup6;
setup6.InitSource(topicName);
- setup6.LoadSource(state3);
-
- auto result = setup6.SourceReadUntil<TString>(UVParser, 1);
- std::vector<TString> data {"yetAnotherData"};
- UNIT_ASSERT_EQUAL(result, data);
- }
- }
-
- Y_UNIT_TEST(LoadCorruptedState) {
+ setup6.LoadSource(state3);
+
+ auto result = setup6.SourceReadUntil<TString>(UVParser, 1);
+ std::vector<TString> data {"yetAnotherData"};
+ UNIT_ASSERT_EQUAL(result, data);
+ }
+ }
+
+ Y_UNIT_TEST(LoadCorruptedState) {
NDqProto::TSourceState state;
- const TString topicName = "Invalid"; // We wouldn't read from this topic.
- auto checkpoint = CreateCheckpoint();
-
- {
+ const TString topicName = "Invalid"; // We wouldn't read from this topic.
+ auto checkpoint = CreateCheckpoint();
+
+ {
TPqIoTestFixture setup1;
setup1.InitSource(topicName);
setup1.SaveSourceState(checkpoint, state);
- }
-
- // Corrupt state.
+ }
+
+ // Corrupt state.
TString corruptedBlob = state.GetData(0).GetStateData().GetBlob();
corruptedBlob.append('a');
state.MutableData(0)->MutableStateData()->SetBlob(corruptedBlob);
-
- {
+
+ {
TPqIoTestFixture setup2;
setup2.InitSource(topicName);
- UNIT_ASSERT_EXCEPTION_CONTAINS(setup2.LoadSource(state), yexception, "Serialized state is corrupted");
- }
- }
+ UNIT_ASSERT_EXCEPTION_CONTAINS(setup2.LoadSource(state), yexception, "Serialized state is corrupted");
+ }
+ }
Y_UNIT_TEST(TestLoadFromSeveralStates) {
const TString topicName = "LoadFromSeveralStates";
@@ -209,5 +209,5 @@ Y_UNIT_TEST_SUITE(TDqPqReadActorTest) {
std::vector<TString> dataResult {"data2", "data3"};
UNIT_ASSERT_EQUAL(result, dataResult);
}
-}
-} // namespace NKikimr::NMiniKQL
+}
+} // namespace NKikimr::NMiniKQL
diff --git a/ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp b/ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp
index 9b7fc77d92e..b64c7ee0e35 100644
--- a/ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp
+++ b/ydb/library/yql/providers/pq/async_io/ut/dq_pq_write_actor_ut.cpp
@@ -1,139 +1,139 @@
-#include "ut_helpers.h"
-
+#include "ut_helpers.h"
+
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NYql::NDq {
-
-constexpr TDuration WaitTimeout = TDuration::MilliSeconds(10000);
-
-Y_UNIT_TEST_SUITE(TPqWriterTest) {
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NYql::NDq {
+
+constexpr TDuration WaitTimeout = TDuration::MilliSeconds(10000);
+
+Y_UNIT_TEST_SUITE(TPqWriterTest) {
Y_UNIT_TEST_F(TestWriteToTopic, TPqIoTestFixture) {
- const TString topicName = "WriteToTopic";
+ const TString topicName = "WriteToTopic";
InitSink(topicName);
-
- const std::vector<TString> data = { "1", "2", "3", "4" };
-
+
+ const std::vector<TString> data = { "1", "2", "3", "4" };
+
SinkWrite(data);
- auto result = PQReadUntil(topicName, 4);
- UNIT_ASSERT_EQUAL(result, data);
- }
-
+ auto result = PQReadUntil(topicName, 4);
+ UNIT_ASSERT_EQUAL(result, data);
+ }
+
Y_UNIT_TEST_F(TestWriteToTopicMultiBatch, TPqIoTestFixture) {
- const TString topicName = "WriteToTopicMultiBatch";
+ const TString topicName = "WriteToTopicMultiBatch";
InitSink(topicName);
-
- const std::vector<TString> data1 = { "1" };
- const std::vector<TString> data2 = { "2" };
- const std::vector<TString> data3 = { "3" };
-
+
+ const std::vector<TString> data1 = { "1" };
+ const std::vector<TString> data2 = { "2" };
+ const std::vector<TString> data3 = { "3" };
+
SinkWrite(data1);
SinkWrite(data2);
SinkWrite(data3);
- auto result = PQReadUntil(topicName, 3);
-
- std::vector<TString> expected = { "1", "2", "3" };
- UNIT_ASSERT_EQUAL(result, expected);
- }
-
+ auto result = PQReadUntil(topicName, 3);
+
+ std::vector<TString> expected = { "1", "2", "3" };
+ UNIT_ASSERT_EQUAL(result, expected);
+ }
+
Y_UNIT_TEST_F(TestDeferredWriteToTopic, TPqIoTestFixture) {
- // In this case we are checking free space overflow
- const TString topicName = "DeferredWriteToTopic";
+ // In this case we are checking free space overflow
+ const TString topicName = "DeferredWriteToTopic";
InitSink(topicName, 1);
-
- const std::vector<TString> data = { "1", "2", "3" };
-
+
+ const std::vector<TString> data = { "1", "2", "3" };
+
auto future = CaSetup->SinkPromises.ResumeExecution.GetFuture();
SinkWrite(data);
- auto result = PQReadUntil(topicName, 3);
-
- UNIT_ASSERT_EQUAL(result, data);
- UNIT_ASSERT(future.Wait(WaitTimeout)); // Resume execution should be called
-
- const std::vector<TString> data2 = { "4", "5", "6" };
-
+ auto result = PQReadUntil(topicName, 3);
+
+ UNIT_ASSERT_EQUAL(result, data);
+ UNIT_ASSERT(future.Wait(WaitTimeout)); // Resume execution should be called
+
+ const std::vector<TString> data2 = { "4", "5", "6" };
+
SinkWrite(data2);
- auto result2 = PQReadUntil(topicName, 6);
- const std::vector<TString> expected = { "1", "2", "3", "4", "5", "6" };
- UNIT_ASSERT_EQUAL(result2, expected);
- }
-
+ auto result2 = PQReadUntil(topicName, 6);
+ const std::vector<TString> expected = { "1", "2", "3", "4", "5", "6" };
+ UNIT_ASSERT_EQUAL(result2, expected);
+ }
+
Y_UNIT_TEST_F(WriteNonExistentTopic, TPqIoTestFixture) {
- const TString topicName = "NonExistentTopic";
+ const TString topicName = "NonExistentTopic";
InitSink(topicName);
-
- const std::vector<TString> data = { "1" };
+
+ const std::vector<TString> data = { "1" };
auto future = CaSetup->SinkPromises.Issue.GetFuture();
SinkWrite(data);
-
- UNIT_ASSERT(future.Wait(WaitTimeout));
+
+ UNIT_ASSERT(future.Wait(WaitTimeout));
UNIT_ASSERT_STRING_CONTAINS(future.GetValue().ToString(), "Write session to topic \"NonExistentTopic\" was closed");
- }
-
- Y_UNIT_TEST(TestCheckpoints) {
- const TString topicName = "Checkpoints";
-
+ }
+
+ Y_UNIT_TEST(TestCheckpoints) {
+ const TString topicName = "Checkpoints";
+
NDqProto::TSinkState state1;
- {
+ {
TPqIoTestFixture setup;
setup.InitSink(topicName);
-
- const std::vector<TString> data1 = { "1" };
+
+ const std::vector<TString> data1 = { "1" };
setup.SinkWrite(data1);
-
- const std::vector<TString> data2 = { "2", "3" };
- auto checkpoint = CreateCheckpoint();
+
+ const std::vector<TString> data2 = { "2", "3" };
+ auto checkpoint = CreateCheckpoint();
auto future = setup.CaSetup->SinkPromises.StateSaved.GetFuture();
setup.SinkWrite(data2, checkpoint);
-
- UNIT_ASSERT(future.Wait(WaitTimeout));
- state1 = future.GetValue();
- }
-
- {
+
+ UNIT_ASSERT(future.Wait(WaitTimeout));
+ state1 = future.GetValue();
+ }
+
+ {
TPqIoTestFixture setup;
setup.InitSink(topicName);
- setup.LoadSink(state1);
-
- const std::vector<TString> data3 = { "4", "5" };
+ setup.LoadSink(state1);
+
+ const std::vector<TString> data3 = { "4", "5" };
setup.SinkWrite(data3);
-
- auto result = PQReadUntil(topicName, 5);
- const std::vector<TString> expected = { "1", "2", "3", "4", "5" };
- UNIT_ASSERT_EQUAL(result, expected);
- }
-
- {
+
+ auto result = PQReadUntil(topicName, 5);
+ const std::vector<TString> expected = { "1", "2", "3", "4", "5" };
+ UNIT_ASSERT_EQUAL(result, expected);
+ }
+
+ {
TPqIoTestFixture setup;
setup.InitSink(topicName);
- setup.LoadSink(state1);
-
- const std::vector<TString> data4 = { "4", "5" };
+ setup.LoadSink(state1);
+
+ const std::vector<TString> data4 = { "4", "5" };
setup.SinkWrite(data4); // This write should be deduplicated
-
- auto result = PQReadUntil(topicName, 4);
- const std::vector<TString> expected = { "1", "2", "3", "4", "5" };
- UNIT_ASSERT_EQUAL(result, expected);
- }
- }
-
+
+ auto result = PQReadUntil(topicName, 4);
+ const std::vector<TString> expected = { "1", "2", "3", "4", "5" };
+ UNIT_ASSERT_EQUAL(result, expected);
+ }
+ }
+
Y_UNIT_TEST_F(TestCheckpointWithEmptyBatch, TPqIoTestFixture) {
- const TString topicName = "Checkpoints";
-
+ const TString topicName = "Checkpoints";
+
NDqProto::TSinkState state1;
- {
+ {
InitSink(topicName);
-
- const std::vector<TString> data = {};
- auto checkpoint = CreateCheckpoint();
+
+ const std::vector<TString> data = {};
+ auto checkpoint = CreateCheckpoint();
auto future = CaSetup->SinkPromises.StateSaved.GetFuture();
SinkWrite(data, checkpoint);
-
- UNIT_ASSERT(future.Wait(WaitTimeout));
- state1 = future.GetValue();
- }
- }
-}
-
-} // NYql::NDq
+
+ UNIT_ASSERT(future.Wait(WaitTimeout));
+ state1 = future.GetValue();
+ }
+ }
+}
+
+} // NYql::NDq
diff --git a/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp b/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp
index 37d8af92eab..6fcae4e0a7c 100644
--- a/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp
+++ b/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.cpp
@@ -1,37 +1,37 @@
-#include "ut_helpers.h"
-
+#include "ut_helpers.h"
+
#include <ydb/library/yql/minikql/mkql_string_util.h>
-
+
#include <ydb/core/testlib/basics/appdata.h>
-
-#include <util/system/env.h>
-
-#include <condition_variable>
-#include <thread>
-
-namespace NYql::NDq {
-
-using namespace NActors;
-
-NYql::NPq::NProto::TDqPqTopicSource BuildPqTopicSourceSettings(TString topic) {
- NYql::NPq::NProto::TDqPqTopicSource settings;
- settings.SetTopicPath(topic);
- settings.SetConsumerName(DefaultPqConsumer);
- settings.SetEndpoint(GetDefaultPqEndpoint());
- settings.MutableToken()->SetName("token");
-
- return settings;
-}
-
-NYql::NPq::NProto::TDqPqTopicSink BuildPqTopicSinkSettings(TString topic) {
- NYql::NPq::NProto::TDqPqTopicSink settings;
- settings.SetTopicPath(topic);
- settings.SetEndpoint(GetDefaultPqEndpoint());
- settings.MutableToken()->SetName("token");
-
- return settings;
-}
-
+
+#include <util/system/env.h>
+
+#include <condition_variable>
+#include <thread>
+
+namespace NYql::NDq {
+
+using namespace NActors;
+
+NYql::NPq::NProto::TDqPqTopicSource BuildPqTopicSourceSettings(TString topic) {
+ NYql::NPq::NProto::TDqPqTopicSource settings;
+ settings.SetTopicPath(topic);
+ settings.SetConsumerName(DefaultPqConsumer);
+ settings.SetEndpoint(GetDefaultPqEndpoint());
+ settings.MutableToken()->SetName("token");
+
+ return settings;
+}
+
+NYql::NPq::NProto::TDqPqTopicSink BuildPqTopicSinkSettings(TString topic) {
+ NYql::NPq::NProto::TDqPqTopicSink settings;
+ settings.SetTopicPath(topic);
+ settings.SetEndpoint(GetDefaultPqEndpoint());
+ settings.MutableToken()->SetName("token");
+
+ return settings;
+}
+
TPqIoTestFixture::TPqIoTestFixture() {
}
@@ -41,144 +41,144 @@ TPqIoTestFixture::~TPqIoTestFixture() {
}
void TPqIoTestFixture::InitSource(
- NYql::NPq::NProto::TDqPqTopicSource&& settings,
- i64 freeSpace)
-{
+ NYql::NPq::NProto::TDqPqTopicSource&& settings,
+ i64 freeSpace)
+{
CaSetup->Execute([&](TFakeActor& actor) {
- NPq::NProto::TDqReadTaskParams params;
- auto* partitioninigParams = params.MutablePartitioningParams();
- partitioninigParams->SetTopicPartitionsCount(1);
- partitioninigParams->SetEachTopicPartitionGroupId(0);
- partitioninigParams->SetDqPartitionsCount(1);
-
- TString serializedParams;
+ NPq::NProto::TDqReadTaskParams params;
+ auto* partitioninigParams = params.MutablePartitioningParams();
+ partitioninigParams->SetTopicPartitionsCount(1);
+ partitioninigParams->SetEachTopicPartitionGroupId(0);
+ partitioninigParams->SetDqPartitionsCount(1);
+
+ TString serializedParams;
Y_PROTOBUF_SUPPRESS_NODISCARD params.SerializeToString(&serializedParams);
-
- const THashMap<TString, TString> secureParams;
- const THashMap<TString, TString> taskParams { {"pq", serializedParams} };
-
- auto [dqSource, dqSourceAsActor] = CreateDqPqReadActor(
- std::move(settings),
- 0,
- "query_1",
- secureParams,
- taskParams,
+
+ const THashMap<TString, TString> secureParams;
+ const THashMap<TString, TString> taskParams { {"pq", serializedParams} };
+
+ auto [dqSource, dqSourceAsActor] = CreateDqPqReadActor(
+ std::move(settings),
+ 0,
+ "query_1",
+ secureParams,
+ taskParams,
Driver,
- nullptr,
- &actor.GetSourceCallbacks(),
- actor.GetHolderFactory(),
- freeSpace);
-
- actor.InitSource(dqSource, dqSourceAsActor);
- });
-}
-
+ nullptr,
+ &actor.GetSourceCallbacks(),
+ actor.GetHolderFactory(),
+ freeSpace);
+
+ actor.InitSource(dqSource, dqSourceAsActor);
+ });
+}
+
void TPqIoTestFixture::InitSink(
- NPq::NProto::TDqPqTopicSink&& settings,
- i64 freeSpace)
-{
- const THashMap<TString, TString> secureParams;
-
+ NPq::NProto::TDqPqTopicSink&& settings,
+ i64 freeSpace)
+{
+ const THashMap<TString, TString> secureParams;
+
CaSetup->Execute([&](TFakeActor& actor) {
- auto [dqSink, dqSinkAsActor] = CreateDqPqWriteActor(
- std::move(settings),
- 0,
- "query_1",
- secureParams,
+ auto [dqSink, dqSinkAsActor] = CreateDqPqWriteActor(
+ std::move(settings),
+ 0,
+ "query_1",
+ secureParams,
Driver,
- nullptr,
- &actor.GetSinkCallbacks(),
- freeSpace);
-
- actor.InitSink(dqSink, dqSinkAsActor);
- });
-}
-
-TString GetDefaultPqEndpoint() {
- auto port = GetEnv("LOGBROKER_PORT");
- UNIT_ASSERT_C(port, "Logbroker recipe is expected");
- return TStringBuilder() << "localhost:" << port;
-}
-
-extern const TString DefaultPqConsumer = "test_client";
-
-void PQWrite(
- const std::vector<TString>& sequence,
- const TString& topic,
- const TString& endpoint)
-{
- NYdb::TDriverConfig cfg;
- cfg.SetEndpoint(endpoint);
- cfg.SetDatabase("/Root");
- cfg.SetLog(CreateLogBackend("cerr"));
- NYdb::TDriver driver(cfg);
- NYdb::NPersQueue::TPersQueueClient client(driver);
- NYdb::NPersQueue::TWriteSessionSettings sessionSettings;
- sessionSettings
- .Path(topic)
- .MessageGroupId("src_id");
- auto session = client.CreateSimpleBlockingWriteSession(sessionSettings);
- for (const TString& data : sequence) {
+ nullptr,
+ &actor.GetSinkCallbacks(),
+ freeSpace);
+
+ actor.InitSink(dqSink, dqSinkAsActor);
+ });
+}
+
+TString GetDefaultPqEndpoint() {
+ auto port = GetEnv("LOGBROKER_PORT");
+ UNIT_ASSERT_C(port, "Logbroker recipe is expected");
+ return TStringBuilder() << "localhost:" << port;
+}
+
+extern const TString DefaultPqConsumer = "test_client";
+
+void PQWrite(
+ const std::vector<TString>& sequence,
+ const TString& topic,
+ const TString& endpoint)
+{
+ NYdb::TDriverConfig cfg;
+ cfg.SetEndpoint(endpoint);
+ cfg.SetDatabase("/Root");
+ cfg.SetLog(CreateLogBackend("cerr"));
+ NYdb::TDriver driver(cfg);
+ NYdb::NPersQueue::TPersQueueClient client(driver);
+ NYdb::NPersQueue::TWriteSessionSettings sessionSettings;
+ sessionSettings
+ .Path(topic)
+ .MessageGroupId("src_id");
+ auto session = client.CreateSimpleBlockingWriteSession(sessionSettings);
+ for (const TString& data : sequence) {
UNIT_ASSERT_C(session->Write(data), "Failed to write message with body \"" << data << "\" to topic " << topic);
- Cerr << "Message '" << data << "' was written into topic '" << topic << "'" << Endl;
- }
- session->Close(); // Wait until all data would be written into PQ.
- session = nullptr;
- driver.Stop(true);
-}
-
-std::vector<TString> PQReadUntil(
- const TString& topic,
- ui64 size,
- const TString& endpoint,
- TDuration timeout)
-{
- NYdb::TDriverConfig cfg;
- cfg.SetEndpoint(endpoint);
- cfg.SetDatabase("/Root");
- cfg.SetLog(CreateLogBackend("cerr"));
- NYdb::TDriver driver(cfg);
- NYdb::NPersQueue::TPersQueueClient client(driver);
- NYdb::NPersQueue::TReadSessionSettings sessionSettings;
- sessionSettings
- .AppendTopics(topic)
- .ConsumerName(DefaultPqConsumer);
-
- auto promise = NThreading::NewPromise();
- std::vector<TString> result;
-
- sessionSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) {
- for (const auto& message : ev.GetMessages()) {
- result.emplace_back(message.GetData());
- }
- if (result.size() >= size) {
- promise.SetValue();
- }
- }, false, false);
-
- std::shared_ptr<NYdb::NPersQueue::IReadSession> session = client.CreateReadSession(sessionSettings);
- UNIT_ASSERT(promise.GetFuture().Wait(timeout));
- session->Close(TDuration::Zero());
- session = nullptr;
- driver.Stop(true);
- return result;
-}
-
-std::vector<TString> UVParser(const NUdf::TUnboxedValue& item) {
- return { TString(item.AsStringRef()) };
-}
-
+ Cerr << "Message '" << data << "' was written into topic '" << topic << "'" << Endl;
+ }
+ session->Close(); // Wait until all data would be written into PQ.
+ session = nullptr;
+ driver.Stop(true);
+}
+
+std::vector<TString> PQReadUntil(
+ const TString& topic,
+ ui64 size,
+ const TString& endpoint,
+ TDuration timeout)
+{
+ NYdb::TDriverConfig cfg;
+ cfg.SetEndpoint(endpoint);
+ cfg.SetDatabase("/Root");
+ cfg.SetLog(CreateLogBackend("cerr"));
+ NYdb::TDriver driver(cfg);
+ NYdb::NPersQueue::TPersQueueClient client(driver);
+ NYdb::NPersQueue::TReadSessionSettings sessionSettings;
+ sessionSettings
+ .AppendTopics(topic)
+ .ConsumerName(DefaultPqConsumer);
+
+ auto promise = NThreading::NewPromise();
+ std::vector<TString> result;
+
+ sessionSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) {
+ for (const auto& message : ev.GetMessages()) {
+ result.emplace_back(message.GetData());
+ }
+ if (result.size() >= size) {
+ promise.SetValue();
+ }
+ }, false, false);
+
+ std::shared_ptr<NYdb::NPersQueue::IReadSession> session = client.CreateReadSession(sessionSettings);
+ UNIT_ASSERT(promise.GetFuture().Wait(timeout));
+ session->Close(TDuration::Zero());
+ session = nullptr;
+ driver.Stop(true);
+ return result;
+}
+
+std::vector<TString> UVParser(const NUdf::TUnboxedValue& item) {
+ return { TString(item.AsStringRef()) };
+}
+
void TPqIoTestFixture::SinkWrite(std::vector<TString> data, TMaybe<NDqProto::TCheckpoint> checkpoint) {
CaSetup->SinkWrite([data](NKikimr::NMiniKQL::THolderFactory& factory) {
- NKikimr::NMiniKQL::TUnboxedValueVector batch;
- batch.reserve(data.size());
- for (const auto& item : data) {
+ NKikimr::NMiniKQL::TUnboxedValueVector batch;
+ batch.reserve(data.size());
+ for (const auto& item : data) {
NUdf::TUnboxedValue* unboxedValueForData = nullptr;
- batch.emplace_back(factory.CreateDirectArrayHolder(1, unboxedValueForData));
+ batch.emplace_back(factory.CreateDirectArrayHolder(1, unboxedValueForData));
unboxedValueForData[0] = NKikimr::NMiniKQL::MakeString(NUdf::TStringRef(item.Data(), item.Size()));
- }
-
- return batch;
- }, checkpoint);
-}
-}
+ }
+
+ return batch;
+ }, checkpoint);
+}
+}
diff --git a/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h b/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h
index 59cd28f7170..5d9f3d11424 100644
--- a/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h
+++ b/ydb/library/yql/providers/pq/async_io/ut/ut_helpers.h
@@ -1,5 +1,5 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/common/ut_helpers/dq_fake_ca.h>
#include <ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h>
#include <ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h>
@@ -8,28 +8,28 @@
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h>
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/minikql/mkql_alloc.h>
-
+
#include <ydb/public/sdk/cpp/client/ydb_persqueue_public/persqueue.h>
#include <ydb/core/testlib/basics/runtime.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <chrono>
-#include <queue>
-
-namespace NYql::NDq {
-
-NYql::NPq::NProto::TDqPqTopicSource BuildPqTopicSourceSettings(TString topic);
-
-NYql::NPq::NProto::TDqPqTopicSink BuildPqTopicSinkSettings(TString topic);
-
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <chrono>
+#include <queue>
+
+namespace NYql::NDq {
+
+NYql::NPq::NProto::TDqPqTopicSource BuildPqTopicSourceSettings(TString topic);
+
+NYql::NPq::NProto::TDqPqTopicSink BuildPqTopicSinkSettings(TString topic);
+
struct TPqIoTestFixture : public NUnitTest::TBaseFixture {
std::unique_ptr<TFakeCASetup> CaSetup = std::make_unique<TFakeCASetup>();
NYdb::TDriver Driver = NYdb::TDriver(NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr")));
-
+
TPqIoTestFixture();
~TPqIoTestFixture();
-
+
void InitSource(
NYql::NPq::NProto::TDqPqTopicSource&& settings,
i64 freeSpace = 1_MB);
@@ -83,24 +83,24 @@ struct TPqIoTestFixture : public NUnitTest::TBaseFixture {
void SinkWrite(std::vector<TString> data, TMaybe<NDqProto::TCheckpoint> checkpoint = Nothing());
};
-TString GetDefaultPqEndpoint();
-
-extern const TString DefaultPqConsumer;
-extern const TString DefaultPqCluster;
-
-// Write using YDB driver
-void PQWrite(
- const std::vector<TString>& sequence,
- const TString& topic,
- const TString& endpoint = GetDefaultPqEndpoint());
-
-// Read using YDB driver
-std::vector<TString> PQReadUntil(
- const TString& topic,
- ui64 size,
- const TString& endpoint = GetDefaultPqEndpoint(),
- TDuration timeout = TDuration::MilliSeconds(10000));
-
-std::vector<TString> UVParser(const NUdf::TUnboxedValue& item);
-
-}
+TString GetDefaultPqEndpoint();
+
+extern const TString DefaultPqConsumer;
+extern const TString DefaultPqCluster;
+
+// Write using YDB driver
+void PQWrite(
+ const std::vector<TString>& sequence,
+ const TString& topic,
+ const TString& endpoint = GetDefaultPqEndpoint());
+
+// Read using YDB driver
+std::vector<TString> PQReadUntil(
+ const TString& topic,
+ ui64 size,
+ const TString& endpoint = GetDefaultPqEndpoint(),
+ TDuration timeout = TDuration::MilliSeconds(10000));
+
+std::vector<TString> UVParser(const NUdf::TUnboxedValue& item);
+
+}
diff --git a/ydb/library/yql/providers/pq/async_io/ut/ya.make b/ydb/library/yql/providers/pq/async_io/ut/ya.make
index a6bcd4a304b..1ce7d821550 100644
--- a/ydb/library/yql/providers/pq/async_io/ut/ya.make
+++ b/ydb/library/yql/providers/pq/async_io/ut/ya.make
@@ -1,26 +1,26 @@
UNITTEST_FOR(ydb/library/yql/providers/pq/async_io)
-
-OWNER(
- d-mokhnatkin
+
+OWNER(
+ d-mokhnatkin
g:yq
- g:yql
-)
-
-ENV(YDB_USE_IN_MEMORY_PDISKS=true)
+ g:yql
+)
+
+ENV(YDB_USE_IN_MEMORY_PDISKS=true)
ENV(LOGBROKER_CREATE_TOPICS=ReadFromTopic,ReadWithFreeSpace,SaveLoadPqRead,WriteToTopic,WriteToTopicMultiBatch,DeferredWriteToTopic,SaveLoadPqWrite,Checkpoints,LoadFromSeveralStates)
-ENV(LOGBROKER_TOPICS_PARTITIONS=1)
+ENV(LOGBROKER_TOPICS_PARTITIONS=1)
INCLUDE(${ARCADIA_ROOT}/kikimr/yq/tools/lbk_recipe_with_dummy_cm/recipe.inc)
-
-SRCS(
- dq_pq_read_actor_ut.cpp
- dq_pq_write_actor_ut.cpp
- ut_helpers.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ dq_pq_read_actor_ut.cpp
+ dq_pq_write_actor_ut.cpp
+ ut_helpers.cpp
+)
+
+PEERDIR(
ydb/core/testlib/basics
ydb/library/yql/minikql
ydb/library/yql/minikql/computation
@@ -29,8 +29,8 @@ PEERDIR(
ydb/public/sdk/cpp/client/ydb_persqueue_public
ydb/library/yql/providers/common/comp_nodes
ydb/library/yql/providers/common/ut_helpers
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/providers/pq/async_io/ya.make b/ydb/library/yql/providers/pq/async_io/ya.make
index cf5ee619971..8c37ecec6fa 100644
--- a/ydb/library/yql/providers/pq/async_io/ya.make
+++ b/ydb/library/yql/providers/pq/async_io/ya.make
@@ -1,18 +1,18 @@
-LIBRARY()
-
-OWNER(
- d-mokhnatkin
+LIBRARY()
+
+OWNER(
+ d-mokhnatkin
g:yq
- g:yql
-)
-
-SRCS(
- dq_pq_read_actor.cpp
- dq_pq_write_actor.cpp
- probes.cpp
-)
-
-PEERDIR(
+ g:yql
+)
+
+SRCS(
+ dq_pq_read_actor.cpp
+ dq_pq_write_actor.cpp
+ probes.cpp
+)
+
+PEERDIR(
ydb/library/yql/minikql/computation
ydb/library/yql/providers/common/token_accessor/client
ydb/library/yql/public/types
@@ -22,12 +22,12 @@ PEERDIR(
ydb/public/sdk/cpp/client/ydb_types/credentials
ydb/library/yql/dq/actors/compute
ydb/library/yql/providers/pq/proto
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
-
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
IF (NOT OPENSOURCE)
IF (OS_LINUX)
# Logbroker recipe is supported only for linux.
diff --git a/ydb/library/yql/providers/pq/cm_client/lib/client.cpp b/ydb/library/yql/providers/pq/cm_client/lib/client.cpp
index 0391d1af902..71d878f5e57 100644
--- a/ydb/library/yql/providers/pq/cm_client/lib/client.cpp
+++ b/ydb/library/yql/providers/pq/cm_client/lib/client.cpp
@@ -246,7 +246,7 @@ private:
NGrpc::TGRpcClientConfig MakeConnectionConfig() const {
NGrpc::TGRpcClientConfig cfg(Options.GetEndpoint(), Options.GetRequestTimeout());
cfg.CompressionAlgoritm = GRPC_COMPRESS_GZIP;
- cfg.EnableSsl = Options.GetEnableSsl();
+ cfg.EnableSsl = Options.GetEnableSsl();
return cfg;
}
diff --git a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp
index e2b1e741a8b..cd536762e4a 100644
--- a/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp
+++ b/ydb/library/yql/providers/pq/gateway/native/yql_pq_session.cpp
@@ -9,8 +9,8 @@ NPq::NConfigurationManager::TClientOptions GetCmClientOptions(const NYql::TPqClu
NPq::NConfigurationManager::TClientOptions opts;
opts
.SetEndpoint(cfg.GetConfigManagerEndpoint())
- .SetCredentialsProviderFactory(credentialsProviderFactory)
- .SetEnableSsl(cfg.GetUseSsl());
+ .SetCredentialsProviderFactory(credentialsProviderFactory)
+ .SetEnableSsl(cfg.GetUseSsl());
return opts;
}
@@ -19,7 +19,7 @@ NYdb::NPersQueue::TPersQueueClientSettings GetYdbPqClientOptions(const TString&
NYdb::NPersQueue::TPersQueueClientSettings opts;
opts
.DiscoveryEndpoint(cfg.GetEndpoint())
- .Database(database)
+ .Database(database)
.EnableSsl(cfg.GetUseSsl())
.CredentialsProviderFactory(credentialsProviderFactory);
diff --git a/ydb/library/yql/providers/pq/proto/dq_io.proto b/ydb/library/yql/providers/pq/proto/dq_io.proto
index 4f86385353c..e87eb72755c 100644
--- a/ydb/library/yql/providers/pq/proto/dq_io.proto
+++ b/ydb/library/yql/providers/pq/proto/dq_io.proto
@@ -1,37 +1,37 @@
-syntax = "proto3";
-option cc_enable_arenas = true;
-
-package NYql.NPq.NProto;
-
-message TToken {
- string Name = 1; // Key in secure params.
- // TODO: token type (oauth, tvm, iam etc).
-}
-
-enum EClusterType {
- Unspecified = 0;
- PersQueue = 1;
- DataStreams = 2;
-}
-
-message TDqPqTopicSource {
- string TopicPath = 1;
- string ConsumerName = 2;
- string Endpoint = 3;
- TToken Token = 4;
- string Database = 5;
- EClusterType ClusterType = 6;
+syntax = "proto3";
+option cc_enable_arenas = true;
+
+package NYql.NPq.NProto;
+
+message TToken {
+ string Name = 1; // Key in secure params.
+ // TODO: token type (oauth, tvm, iam etc).
+}
+
+enum EClusterType {
+ Unspecified = 0;
+ PersQueue = 1;
+ DataStreams = 2;
+}
+
+message TDqPqTopicSource {
+ string TopicPath = 1;
+ string ConsumerName = 2;
+ string Endpoint = 3;
+ TToken Token = 4;
+ string Database = 5;
+ EClusterType ClusterType = 6;
bool UseSsl = 7;
bool AddBearerToToken = 8;
string DatabaseId = 9;
-}
-
-message TDqPqTopicSink {
- string TopicPath = 1;
- string Endpoint = 2;
- TToken Token = 3;
- string Database = 4;
+}
+
+message TDqPqTopicSink {
+ string TopicPath = 1;
+ string Endpoint = 2;
+ TToken Token = 3;
+ string Database = 4;
bool UseSsl = 5;
bool AddBearerToToken = 6;
- EClusterType ClusterType = 7;
-}
+ EClusterType ClusterType = 7;
+}
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp
index ced4c9dc98e..ea67ffd422c 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp
@@ -131,14 +131,14 @@ public:
return {};
}
- static NPq::NProto::EClusterType ToClusterType(NYql::TPqClusterConfig::EClusterType t) {
+ static NPq::NProto::EClusterType ToClusterType(NYql::TPqClusterConfig::EClusterType t) {
switch (t) {
case NYql::TPqClusterConfig::CT_UNSPECIFIED:
- return NPq::NProto::Unspecified;
+ return NPq::NProto::Unspecified;
case NYql::TPqClusterConfig::CT_PERS_QUEUE:
- return NPq::NProto::PersQueue;
+ return NPq::NProto::PersQueue;
case NYql::TPqClusterConfig::CT_DATA_STREAMS:
- return NPq::NProto::DataStreams;
+ return NPq::NProto::DataStreams;
}
}
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp
index 753987fe6a9..8886ddb83e7 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_mkql_compiler.cpp
@@ -13,17 +13,17 @@ void RegisterDqPqMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler) {
compiler.ChainCallable(TDqSourceWideWrap::CallableName(),
[](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) {
if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == PqProviderName) {
- const auto wrapped = TryWrapWithParser(wrapper, ctx);
- if (wrapped) {
- return *wrapped;
+ const auto wrapped = TryWrapWithParser(wrapper, ctx);
+ if (wrapped) {
+ return *wrapped;
}
- const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx);
+ const auto input = MkqlBuildExpr(wrapper.Input().Ref(), ctx);
auto flow = ctx.ProgramBuilder.ToFlow(input);
return ctx.ProgramBuilder.ExpandMap(flow,
- [&](TRuntimeNode item) -> TRuntimeNode::TList {
- return {item};
- });
+ [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {item};
+ });
}
return TRuntimeNode();
diff --git a/ydb/library/yql/providers/pq/ya.make b/ydb/library/yql/providers/pq/ya.make
index f093910cc75..840d37c7d53 100644
--- a/ydb/library/yql/providers/pq/ya.make
+++ b/ydb/library/yql/providers/pq/ya.make
@@ -5,7 +5,7 @@ OWNER(
)
RECURSE(
- async_io
+ async_io
common
cm_client
expr_nodes
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
index 0b25623b96c..eb5410a24e3 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
@@ -190,7 +190,7 @@ private:
return;
}
++IsDoneCounter;
- Callbacks->OnSourceError(InputIndex, result->Get()->Error, true);
+ Callbacks->OnSourceError(InputIndex, result->Get()->Error, true);
}
// IActor & IDqSourceActor
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp
index 5a2f3e100c8..fb782404982 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp
@@ -2,7 +2,7 @@
#include "yql_s3_read_actor.h"
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h>
-
+
namespace NYql::NDq {
void RegisterS3ReadActorFactory(
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp
index cb2822fb765..fab2d6d0f75 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_mkql_compiler.cpp
@@ -14,10 +14,10 @@ void RegisterDqS3MkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, con
compiler.ChainCallable(TDqSourceWideWrap::CallableName(),
[](const TExprNode& node, NCommon::TMkqlBuildContext& ctx) {
if (const auto wrapper = TDqSourceWideWrap(&node); wrapper.DataSource().Category().Value() == S3ProviderName) {
- const auto wrapped = TryWrapWithParser(wrapper, ctx);
- if (wrapped) {
- return *wrapped;
- }
+ const auto wrapped = TryWrapWithParser(wrapper, ctx);
+ if (wrapped) {
+ return *wrapped;
+ }
}
return TRuntimeNode();
diff --git a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp
index fcb919a59e6..fc07635176f 100644
--- a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp
+++ b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.cpp
@@ -1,10 +1,10 @@
-#include "dq_solomon_write_actor.h"
-#include "metrics_encoder.h"
-
+#include "dq_solomon_write_actor.h"
+#include "metrics_encoder.h"
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h>
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h>
-
+
#include <ydb/library/yql/utils/actor_log/log.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/minikql/comp_nodes/mkql_saveload.h>
@@ -12,472 +12,472 @@
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/utils/yql_panic.h>
#include <ydb/library/yql/utils/actors/http_sender_actor.h>
-
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/event_local.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/log.h>
-#include <library/cpp/actors/http/http_proxy.h>
-#include <library/cpp/json/easy_parse/json_easy_parser.h>
-
-
-#include <util/generic/algorithm.h>
-#include <util/generic/hash.h>
-#include <util/system/compiler.h>
-
-#include <algorithm>
-#include <queue>
-#include <variant>
-
-#define SINK_LOG_T(s) \
- LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
-#define SINK_LOG_D(s) \
- LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
-#define SINK_LOG_I(s) \
- LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
-#define SINK_LOG_W(s) \
- LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
-#define SINK_LOG_N(s) \
- LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
-#define SINK_LOG_E(s) \
- LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
-#define SINK_LOG_C(s) \
- LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
-#define SINK_LOG(prio, s) \
- LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
-
-namespace NYql::NDq {
-
-using namespace NActors;
-using namespace NLog;
-using namespace NKikimr::NMiniKQL;
-
-namespace {
-
-const ui64 MaxMetricsPerRequest = 1000; // Max allowed count is 10000
-const ui64 MaxRequestsInflight = 3;
-
-struct TDqSolomonWriteParams {
- NSo::NProto::TDqSolomonShard Shard;
-};
-
-struct TMetricsToSend {
- TString Data;
- ui64 MetricsCount = 0;
-};
-
-struct TMetricsInflight {
- TActorId HttpSenderId;
- ui64 MetricsCount = 0;
-};
-
-} // namespace
-
-class TDqSolomonWriteActor : public NActors::TActor<TDqSolomonWriteActor>, public IDqSinkActor {
-public:
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/event_local.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+#include <library/cpp/actors/http/http_proxy.h>
+#include <library/cpp/json/easy_parse/json_easy_parser.h>
+
+
+#include <util/generic/algorithm.h>
+#include <util/generic/hash.h>
+#include <util/system/compiler.h>
+
+#include <algorithm>
+#include <queue>
+#include <variant>
+
+#define SINK_LOG_T(s) \
+ LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
+#define SINK_LOG_D(s) \
+ LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
+#define SINK_LOG_I(s) \
+ LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
+#define SINK_LOG_W(s) \
+ LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
+#define SINK_LOG_N(s) \
+ LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
+#define SINK_LOG_E(s) \
+ LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
+#define SINK_LOG_C(s) \
+ LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
+#define SINK_LOG(prio, s) \
+ LOG_LOG_S(*NActors::TlsActivationContext, prio, NKikimrServices::KQP_COMPUTE, "Solomon sink. " << s)
+
+namespace NYql::NDq {
+
+using namespace NActors;
+using namespace NLog;
+using namespace NKikimr::NMiniKQL;
+
+namespace {
+
+const ui64 MaxMetricsPerRequest = 1000; // Max allowed count is 10000
+const ui64 MaxRequestsInflight = 3;
+
+struct TDqSolomonWriteParams {
+ NSo::NProto::TDqSolomonShard Shard;
+};
+
+struct TMetricsToSend {
+ TString Data;
+ ui64 MetricsCount = 0;
+};
+
+struct TMetricsInflight {
+ TActorId HttpSenderId;
+ ui64 MetricsCount = 0;
+};
+
+} // namespace
+
+class TDqSolomonWriteActor : public NActors::TActor<TDqSolomonWriteActor>, public IDqSinkActor {
+public:
static constexpr char ActorName[] = "DQ_SOLOMON_WRITE_ACTOR";
- TDqSolomonWriteActor(
- ui64 outputIndex,
- TDqSolomonWriteParams&& writeParams,
- NYql::NDq::IDqSinkActor::ICallbacks* callbacks,
+ TDqSolomonWriteActor(
+ ui64 outputIndex,
+ TDqSolomonWriteParams&& writeParams,
+ NYql::NDq::IDqSinkActor::ICallbacks* callbacks,
const NMonitoring::TDynamicCounterPtr& counters,
std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider,
- i64 freeSpace)
- : TActor<TDqSolomonWriteActor>(&TDqSolomonWriteActor::StateFunc)
- , OutputIndex(outputIndex)
- , WriteParams(std::move(writeParams))
- , Url(GetUrl())
- , Callbacks(callbacks)
- , Metrics(counters)
- , FreeSpace(freeSpace)
- , UserMetricsEncoder(
- WriteParams.Shard.GetScheme(),
- WriteParams.Shard.GetClusterType() == NSo::NProto::ESolomonClusterType::CT_MONITORING)
+ i64 freeSpace)
+ : TActor<TDqSolomonWriteActor>(&TDqSolomonWriteActor::StateFunc)
+ , OutputIndex(outputIndex)
+ , WriteParams(std::move(writeParams))
+ , Url(GetUrl())
+ , Callbacks(callbacks)
+ , Metrics(counters)
+ , FreeSpace(freeSpace)
+ , UserMetricsEncoder(
+ WriteParams.Shard.GetScheme(),
+ WriteParams.Shard.GetClusterType() == NSo::NProto::ESolomonClusterType::CT_MONITORING)
, CredentialsProvider(credentialsProvider)
- {
- SINK_LOG_D("Init");
- }
-
- STRICT_STFUNC(StateFunc,
+ {
+ SINK_LOG_D("Init");
+ }
+
+ STRICT_STFUNC(StateFunc,
hFunc(TEvHttpBase::TEvSendResult, Handle);
- )
-
-public:
- void SendData(
- NKikimr::NMiniKQL::TUnboxedValueVector&& batch,
- i64,
- const TMaybe<NDqProto::TCheckpoint>& checkpoint,
- bool) override
- {
- SINK_LOG_D("Got " << batch.size() << " items to send");
-
- ui64 metricsCount = 0;
- for (const auto& item : batch) {
- if (metricsCount + WriteParams.Shard.GetScheme().GetSensors().size() > MaxMetricsPerRequest) {
- PushMetricsToBuffer(metricsCount);
- }
-
- metricsCount += UserMetricsEncoder.Append(item);
- }
-
- if (metricsCount != 0) {
- PushMetricsToBuffer(metricsCount);
- }
-
- if (checkpoint) {
- SendingBuffer.emplace(*checkpoint);
- }
-
- while (TryToSendNextBatch()) {}
-
- if (FreeSpace <= 0) {
- ShouldNotifyNewFreeSpace = true;
- }
- };
-
+ )
+
+public:
+ void SendData(
+ NKikimr::NMiniKQL::TUnboxedValueVector&& batch,
+ i64,
+ const TMaybe<NDqProto::TCheckpoint>& checkpoint,
+ bool) override
+ {
+ SINK_LOG_D("Got " << batch.size() << " items to send");
+
+ ui64 metricsCount = 0;
+ for (const auto& item : batch) {
+ if (metricsCount + WriteParams.Shard.GetScheme().GetSensors().size() > MaxMetricsPerRequest) {
+ PushMetricsToBuffer(metricsCount);
+ }
+
+ metricsCount += UserMetricsEncoder.Append(item);
+ }
+
+ if (metricsCount != 0) {
+ PushMetricsToBuffer(metricsCount);
+ }
+
+ if (checkpoint) {
+ SendingBuffer.emplace(*checkpoint);
+ }
+
+ while (TryToSendNextBatch()) {}
+
+ if (FreeSpace <= 0) {
+ ShouldNotifyNewFreeSpace = true;
+ }
+ };
+
void LoadState(const NDqProto::TSinkState&) override { }
-
- void CommitState(const NDqProto::TCheckpoint&) override { }
-
- i64 GetFreeSpace() const override {
- return FreeSpace;
- };
-
- ui64 GetOutputIndex() const override {
- return OutputIndex;
- };
-
-private:
- struct TDqSolomonWriteActorMetrics {
- explicit TDqSolomonWriteActorMetrics(const NMonitoring::TDynamicCounterPtr& counters) {
- auto subgroup = counters->GetSubgroup("subsystem", "dq_solomon_write_actor");
- SendingBufferSize = subgroup->GetCounter("SendingBufferSize");
- WindowMinSendingBufferSize = subgroup->GetCounter("WindowMinSendingBufferSize");
- InflightRequests = subgroup->GetCounter("InflightRequests");
- WindowMinInflightRequests = subgroup->GetCounter("WindowMinInflightRequests");
- SentMetrics = subgroup->GetCounter("SentMetrics", true);
- СonfirmedMetrics = subgroup->GetCounter("СonfirmedMetrics", true);
- }
-
- NMonitoring::TDynamicCounters::TCounterPtr SendingBufferSize;
- NMonitoring::TDynamicCounters::TCounterPtr WindowMinSendingBufferSize;
- NMonitoring::TDynamicCounters::TCounterPtr InflightRequests;
- NMonitoring::TDynamicCounters::TCounterPtr WindowMinInflightRequests;
- NMonitoring::TDynamicCounters::TCounterPtr SentMetrics;
- NMonitoring::TDynamicCounters::TCounterPtr СonfirmedMetrics;
-
- public:
- void ReportSendingBufferSize(size_t size) {
- SendingBufferSizeHistoryMin += size;
- SendingBufferSizeHistory.push(size);
-
- if (SendingBufferSizeHistory.size() > WindowSize) {
- SendingBufferSizeHistoryMin -= SendingBufferSizeHistory.front();
- SendingBufferSizeHistory.pop();
- }
-
- *SendingBufferSize = size;
- *WindowMinSendingBufferSize = SendingBufferSizeHistoryMin / SendingBufferSizeHistory.size();
- }
-
- void ReportInflightRequestsCount(size_t count) {
- InflightRequestsHistoryMin += count;
- InflightRequestsHistory.push(count);
-
- if (InflightRequestsHistory.size() > WindowSize) {
- InflightRequestsHistoryMin -= InflightRequestsHistory.front();
- InflightRequestsHistory.pop();
- }
-
- *InflightRequests = count;
- *WindowMinInflightRequests = InflightRequestsHistoryMin / InflightRequestsHistory.size();
- }
-
- private:
- constexpr static size_t WindowSize = 30;
-
- std::queue<size_t> SendingBufferSizeHistory;
- size_t SendingBufferSizeHistoryMin = 0;
-
- std::queue<size_t> InflightRequestsHistory;
- size_t InflightRequestsHistoryMin = 0;
- };
-
+
+ void CommitState(const NDqProto::TCheckpoint&) override { }
+
+ i64 GetFreeSpace() const override {
+ return FreeSpace;
+ };
+
+ ui64 GetOutputIndex() const override {
+ return OutputIndex;
+ };
+
+private:
+ struct TDqSolomonWriteActorMetrics {
+ explicit TDqSolomonWriteActorMetrics(const NMonitoring::TDynamicCounterPtr& counters) {
+ auto subgroup = counters->GetSubgroup("subsystem", "dq_solomon_write_actor");
+ SendingBufferSize = subgroup->GetCounter("SendingBufferSize");
+ WindowMinSendingBufferSize = subgroup->GetCounter("WindowMinSendingBufferSize");
+ InflightRequests = subgroup->GetCounter("InflightRequests");
+ WindowMinInflightRequests = subgroup->GetCounter("WindowMinInflightRequests");
+ SentMetrics = subgroup->GetCounter("SentMetrics", true);
+ СonfirmedMetrics = subgroup->GetCounter("СonfirmedMetrics", true);
+ }
+
+ NMonitoring::TDynamicCounters::TCounterPtr SendingBufferSize;
+ NMonitoring::TDynamicCounters::TCounterPtr WindowMinSendingBufferSize;
+ NMonitoring::TDynamicCounters::TCounterPtr InflightRequests;
+ NMonitoring::TDynamicCounters::TCounterPtr WindowMinInflightRequests;
+ NMonitoring::TDynamicCounters::TCounterPtr SentMetrics;
+ NMonitoring::TDynamicCounters::TCounterPtr СonfirmedMetrics;
+
+ public:
+ void ReportSendingBufferSize(size_t size) {
+ SendingBufferSizeHistoryMin += size;
+ SendingBufferSizeHistory.push(size);
+
+ if (SendingBufferSizeHistory.size() > WindowSize) {
+ SendingBufferSizeHistoryMin -= SendingBufferSizeHistory.front();
+ SendingBufferSizeHistory.pop();
+ }
+
+ *SendingBufferSize = size;
+ *WindowMinSendingBufferSize = SendingBufferSizeHistoryMin / SendingBufferSizeHistory.size();
+ }
+
+ void ReportInflightRequestsCount(size_t count) {
+ InflightRequestsHistoryMin += count;
+ InflightRequestsHistory.push(count);
+
+ if (InflightRequestsHistory.size() > WindowSize) {
+ InflightRequestsHistoryMin -= InflightRequestsHistory.front();
+ InflightRequestsHistory.pop();
+ }
+
+ *InflightRequests = count;
+ *WindowMinInflightRequests = InflightRequestsHistoryMin / InflightRequestsHistory.size();
+ }
+
+ private:
+ constexpr static size_t WindowSize = 30;
+
+ std::queue<size_t> SendingBufferSizeHistory;
+ size_t SendingBufferSizeHistoryMin = 0;
+
+ std::queue<size_t> InflightRequestsHistory;
+ size_t InflightRequestsHistoryMin = 0;
+ };
+
void Handle(TEvHttpBase::TEvSendResult::TPtr& ev) {
- const auto* res = ev->Get();
- const TString& error = res->HttpIncomingResponse->Get()->GetError();
-
- if (!error.empty()) {
- TStringBuilder errorBuilder;
- errorBuilder << "Error while sending request to monitoring api: " << error;
- const auto& response = res->HttpIncomingResponse->Get()->Response;
- if (response) {
- errorBuilder << " " << response->GetObfuscatedData();
- }
-
- TIssues issues { TIssue(errorBuilder) };
- SINK_LOG_W("Got error response from solomon " << issues.ToString());
- Callbacks->OnSinkError(OutputIndex, issues, res->IsTerminal);
- return;
- }
-
- HandleSuccessSolomonResponse(*res->HttpIncomingResponse->Get());
-
- while (TryToSendNextBatch()) {}
- }
-
+ const auto* res = ev->Get();
+ const TString& error = res->HttpIncomingResponse->Get()->GetError();
+
+ if (!error.empty()) {
+ TStringBuilder errorBuilder;
+ errorBuilder << "Error while sending request to monitoring api: " << error;
+ const auto& response = res->HttpIncomingResponse->Get()->Response;
+ if (response) {
+ errorBuilder << " " << response->GetObfuscatedData();
+ }
+
+ TIssues issues { TIssue(errorBuilder) };
+ SINK_LOG_W("Got error response from solomon " << issues.ToString());
+ Callbacks->OnSinkError(OutputIndex, issues, res->IsTerminal);
+ return;
+ }
+
+ HandleSuccessSolomonResponse(*res->HttpIncomingResponse->Get());
+
+ while (TryToSendNextBatch()) {}
+ }
+
// IActor & IDqSinkActor
void PassAway() override { // Is called from Compute Actor
- for (const auto& [_, metricsInflight] : InflightBuffer) {
- Send(metricsInflight.HttpSenderId, new TEvents::TEvPoison());
- }
-
- if (HttpProxyId) {
- Send(HttpProxyId, new TEvents::TEvPoison());
- }
-
+ for (const auto& [_, metricsInflight] : InflightBuffer) {
+ Send(metricsInflight.HttpSenderId, new TEvents::TEvPoison());
+ }
+
+ if (HttpProxyId) {
+ Send(HttpProxyId, new TEvents::TEvPoison());
+ }
+
TActor<TDqSolomonWriteActor>::PassAway();
- }
-
-private:
+ }
+
+private:
NDqProto::TSinkState BuildState() { return {}; }
-
- TString GetUrl() const {
- TStringBuilder builder;
+
+ TString GetUrl() const {
+ TStringBuilder builder;
builder << (WriteParams.Shard.GetUseSsl() ? "https://" : "http://");
- builder << WriteParams.Shard.GetEndpoint();
-
- switch (WriteParams.Shard.GetClusterType()) {
- case NSo::NProto::ESolomonClusterType::CT_SOLOMON: {
- builder << "/api/v2/push";
- builder << "?project=" << WriteParams.Shard.GetProject();
- builder << "&cluster=" << WriteParams.Shard.GetCluster();
- builder << "&service=" << WriteParams.Shard.GetService();
- break;
- }
- case NSo::NProto::ESolomonClusterType::CT_MONITORING: {
- builder << "/monitoring/v2/data/write";
- builder << "?folderId=" << WriteParams.Shard.GetCluster();
- builder << "&service=" << WriteParams.Shard.GetService();
- break;
- }
- default:
- Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType()));
- }
-
- return builder;
- }
-
- void PushMetricsToBuffer(ui64& metricsCount) {
- try {
- auto data = UserMetricsEncoder.Encode();
- SINK_LOG_D("Push " << data.size() << " bytes of data to buffer");
-
- FreeSpace -= data.size();
- SendingBuffer.emplace(TMetricsToSend { std::move(data), metricsCount });
- } catch (const yexception& e) {
- TIssues issues { TIssue(TStringBuilder() << "Error while encoding solomon metrics: " << e.what()) };
- Callbacks->OnSinkError(OutputIndex, issues, true);
- }
-
- metricsCount = 0;
- }
-
- NHttp::THttpOutgoingRequestPtr BuildSolomonRequest(const TString& data) {
- NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestPost(Url);
- FillAuth(httpRequest);
- httpRequest->Set<&NHttp::THttpRequest::ContentType>("application/json");
- httpRequest->Set<&NHttp::THttpRequest::Body>(data);
- return httpRequest;
- }
-
- void FillAuth(NHttp::THttpOutgoingRequestPtr& httpRequest) {
- const TString authorizationHeader = "Authorization";
+ builder << WriteParams.Shard.GetEndpoint();
+
+ switch (WriteParams.Shard.GetClusterType()) {
+ case NSo::NProto::ESolomonClusterType::CT_SOLOMON: {
+ builder << "/api/v2/push";
+ builder << "?project=" << WriteParams.Shard.GetProject();
+ builder << "&cluster=" << WriteParams.Shard.GetCluster();
+ builder << "&service=" << WriteParams.Shard.GetService();
+ break;
+ }
+ case NSo::NProto::ESolomonClusterType::CT_MONITORING: {
+ builder << "/monitoring/v2/data/write";
+ builder << "?folderId=" << WriteParams.Shard.GetCluster();
+ builder << "&service=" << WriteParams.Shard.GetService();
+ break;
+ }
+ default:
+ Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType()));
+ }
+
+ return builder;
+ }
+
+ void PushMetricsToBuffer(ui64& metricsCount) {
+ try {
+ auto data = UserMetricsEncoder.Encode();
+ SINK_LOG_D("Push " << data.size() << " bytes of data to buffer");
+
+ FreeSpace -= data.size();
+ SendingBuffer.emplace(TMetricsToSend { std::move(data), metricsCount });
+ } catch (const yexception& e) {
+ TIssues issues { TIssue(TStringBuilder() << "Error while encoding solomon metrics: " << e.what()) };
+ Callbacks->OnSinkError(OutputIndex, issues, true);
+ }
+
+ metricsCount = 0;
+ }
+
+ NHttp::THttpOutgoingRequestPtr BuildSolomonRequest(const TString& data) {
+ NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestPost(Url);
+ FillAuth(httpRequest);
+ httpRequest->Set<&NHttp::THttpRequest::ContentType>("application/json");
+ httpRequest->Set<&NHttp::THttpRequest::Body>(data);
+ return httpRequest;
+ }
+
+ void FillAuth(NHttp::THttpOutgoingRequestPtr& httpRequest) {
+ const TString authorizationHeader = "Authorization";
const TString authToken = CredentialsProvider->GetAuthInfo();
-
- switch (WriteParams.Shard.GetClusterType()) {
- case NSo::NProto::ESolomonClusterType::CT_SOLOMON:
+
+ switch (WriteParams.Shard.GetClusterType()) {
+ case NSo::NProto::ESolomonClusterType::CT_SOLOMON:
httpRequest->Set(authorizationHeader, "OAuth " + authToken);
- break;
- case NSo::NProto::ESolomonClusterType::CT_MONITORING:
+ break;
+ case NSo::NProto::ESolomonClusterType::CT_MONITORING:
httpRequest->Set(authorizationHeader, "Bearer " + authToken);
- break;
- default:
- Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType()));
- }
- }
-
- bool TryToSendNextBatch() {
- Metrics.ReportSendingBufferSize(SendingBuffer.size());
- Metrics.ReportInflightRequestsCount(InflightBuffer.size());
-
- if (CheckpointInProgress || SendingBuffer.empty() || InflightBuffer.size() >= MaxRequestsInflight) {
- TStringBuilder skipReason;
- skipReason << "Skip sending to solomon. Reason: ";
- if (CheckpointInProgress) {
- skipReason << "CheckpointInProgress ";
- }
- if (SendingBuffer.empty()) {
- skipReason << "Empty buffer ";
- }
- if (InflightBuffer.size() >= MaxRequestsInflight) {
- skipReason << "MaxRequestsInflight ";
- }
- SINK_LOG_D(skipReason);
- return false;
- }
-
- auto variant = SendingBuffer.front();
- SendingBuffer.pop();
-
- if (std::holds_alternative<TMetricsToSend>(variant)) {
- if (Y_UNLIKELY(!HttpProxyId)) {
- HttpProxyId = Register(NHttp::CreateHttpProxy(*NMonitoring::TMetricRegistry::Instance()));
- }
-
- const auto metricsToSend = std::get<TMetricsToSend>(variant);
- const NHttp::THttpOutgoingRequestPtr httpRequest = BuildSolomonRequest(metricsToSend.Data);
-
- const TActorId httpSenderId = Register(CreateHttpSenderActor(SelfId(), HttpProxyId));
- Send(httpSenderId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest));
- SINK_LOG_D("Sent " << metricsToSend.MetricsCount << " metrics with size of " << metricsToSend.Data.size() << " bytes to solomon");
-
- *Metrics.SentMetrics += metricsToSend.MetricsCount;
- InflightBuffer.emplace(httpRequest, TMetricsInflight { httpSenderId, metricsToSend.MetricsCount });
- return true;
- }
-
- if (std::holds_alternative<NDqProto::TCheckpoint>(variant)) {
- CheckpointInProgress = std::get<NDqProto::TCheckpoint>(std::move(variant));
- if (InflightBuffer.empty()) {
- DoCheckpoint();
- return true;
- }
- return false;
- }
-
- Y_ENSURE(false, "Bad type");
- }
-
- void HandleSuccessSolomonResponse(const NHttp::TEvHttpProxy::TEvHttpIncomingResponse& response) {
- SINK_LOG_D("Solomon response: " << response.Response->GetObfuscatedData());
- NJson::TJsonParser parser;
- switch (WriteParams.Shard.GetClusterType()) {
- case NSo::NProto::ESolomonClusterType::CT_SOLOMON:
- parser.AddField("sensorsProcessed", true);
- break;
- case NSo::NProto::ESolomonClusterType::CT_MONITORING:
- parser.AddField("writtenMetricsCount", true);
- break;
- default:
- Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType()));
- }
- parser.AddField("errorMessage", false);
-
- TVector<TString> res;
- if (!parser.Parse(TString(response.Response->Body), &res)) {
- TIssues issues { TIssue(TStringBuilder() << "Invalid monitoring response: " << response.Response->GetObfuscatedData()) };
- Callbacks->OnSinkError(OutputIndex, issues, true);
- return;
- }
- Y_VERIFY(res.size() == 2);
-
- auto ptr = InflightBuffer.find(response.Request);
- Y_VERIFY(ptr != InflightBuffer.end());
-
- const ui64 writtenMetricsCount = std::stoul(res[0]);
- *Metrics.СonfirmedMetrics += writtenMetricsCount;
- if (writtenMetricsCount != ptr->second.MetricsCount) {
- // TODO: YQ-340
- // TIssues issues { TIssue(TStringBuilder() << ToString(ptr->second.MetricsCount - writtenMetricsCount) << " metrics were not written: " << res[1]) };
- // Callbacks->OnSinkError(OutputIndex, issues, true);
- // return;
- SINK_LOG_W("Some metrics were not written. MetricsCount=" << ptr->second.MetricsCount << " writtenMetricsCount=" << writtenMetricsCount << " Solomon response: " << response.Response->GetObfuscatedData());
- }
-
- FreeSpace += ptr->first->Body.Size();
- if (ShouldNotifyNewFreeSpace) {
- Callbacks->ResumeExecution();
- ShouldNotifyNewFreeSpace = false;
- }
- InflightBuffer.erase(ptr);
-
- if (CheckpointInProgress && InflightBuffer.empty()) {
- DoCheckpoint();
- }
- }
-
- void DoCheckpoint() {
+ break;
+ default:
+ Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType()));
+ }
+ }
+
+ bool TryToSendNextBatch() {
+ Metrics.ReportSendingBufferSize(SendingBuffer.size());
+ Metrics.ReportInflightRequestsCount(InflightBuffer.size());
+
+ if (CheckpointInProgress || SendingBuffer.empty() || InflightBuffer.size() >= MaxRequestsInflight) {
+ TStringBuilder skipReason;
+ skipReason << "Skip sending to solomon. Reason: ";
+ if (CheckpointInProgress) {
+ skipReason << "CheckpointInProgress ";
+ }
+ if (SendingBuffer.empty()) {
+ skipReason << "Empty buffer ";
+ }
+ if (InflightBuffer.size() >= MaxRequestsInflight) {
+ skipReason << "MaxRequestsInflight ";
+ }
+ SINK_LOG_D(skipReason);
+ return false;
+ }
+
+ auto variant = SendingBuffer.front();
+ SendingBuffer.pop();
+
+ if (std::holds_alternative<TMetricsToSend>(variant)) {
+ if (Y_UNLIKELY(!HttpProxyId)) {
+ HttpProxyId = Register(NHttp::CreateHttpProxy(*NMonitoring::TMetricRegistry::Instance()));
+ }
+
+ const auto metricsToSend = std::get<TMetricsToSend>(variant);
+ const NHttp::THttpOutgoingRequestPtr httpRequest = BuildSolomonRequest(metricsToSend.Data);
+
+ const TActorId httpSenderId = Register(CreateHttpSenderActor(SelfId(), HttpProxyId));
+ Send(httpSenderId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest));
+ SINK_LOG_D("Sent " << metricsToSend.MetricsCount << " metrics with size of " << metricsToSend.Data.size() << " bytes to solomon");
+
+ *Metrics.SentMetrics += metricsToSend.MetricsCount;
+ InflightBuffer.emplace(httpRequest, TMetricsInflight { httpSenderId, metricsToSend.MetricsCount });
+ return true;
+ }
+
+ if (std::holds_alternative<NDqProto::TCheckpoint>(variant)) {
+ CheckpointInProgress = std::get<NDqProto::TCheckpoint>(std::move(variant));
+ if (InflightBuffer.empty()) {
+ DoCheckpoint();
+ return true;
+ }
+ return false;
+ }
+
+ Y_ENSURE(false, "Bad type");
+ }
+
+ void HandleSuccessSolomonResponse(const NHttp::TEvHttpProxy::TEvHttpIncomingResponse& response) {
+ SINK_LOG_D("Solomon response: " << response.Response->GetObfuscatedData());
+ NJson::TJsonParser parser;
+ switch (WriteParams.Shard.GetClusterType()) {
+ case NSo::NProto::ESolomonClusterType::CT_SOLOMON:
+ parser.AddField("sensorsProcessed", true);
+ break;
+ case NSo::NProto::ESolomonClusterType::CT_MONITORING:
+ parser.AddField("writtenMetricsCount", true);
+ break;
+ default:
+ Y_ENSURE(false, "Invalid cluster type " << ToString<ui32>(WriteParams.Shard.GetClusterType()));
+ }
+ parser.AddField("errorMessage", false);
+
+ TVector<TString> res;
+ if (!parser.Parse(TString(response.Response->Body), &res)) {
+ TIssues issues { TIssue(TStringBuilder() << "Invalid monitoring response: " << response.Response->GetObfuscatedData()) };
+ Callbacks->OnSinkError(OutputIndex, issues, true);
+ return;
+ }
+ Y_VERIFY(res.size() == 2);
+
+ auto ptr = InflightBuffer.find(response.Request);
+ Y_VERIFY(ptr != InflightBuffer.end());
+
+ const ui64 writtenMetricsCount = std::stoul(res[0]);
+ *Metrics.СonfirmedMetrics += writtenMetricsCount;
+ if (writtenMetricsCount != ptr->second.MetricsCount) {
+ // TODO: YQ-340
+ // TIssues issues { TIssue(TStringBuilder() << ToString(ptr->second.MetricsCount - writtenMetricsCount) << " metrics were not written: " << res[1]) };
+ // Callbacks->OnSinkError(OutputIndex, issues, true);
+ // return;
+ SINK_LOG_W("Some metrics were not written. MetricsCount=" << ptr->second.MetricsCount << " writtenMetricsCount=" << writtenMetricsCount << " Solomon response: " << response.Response->GetObfuscatedData());
+ }
+
+ FreeSpace += ptr->first->Body.Size();
+ if (ShouldNotifyNewFreeSpace) {
+ Callbacks->ResumeExecution();
+ ShouldNotifyNewFreeSpace = false;
+ }
+ InflightBuffer.erase(ptr);
+
+ if (CheckpointInProgress && InflightBuffer.empty()) {
+ DoCheckpoint();
+ }
+ }
+
+ void DoCheckpoint() {
Callbacks->OnSinkStateSaved(BuildState(), OutputIndex, *CheckpointInProgress);
- CheckpointInProgress = std::nullopt;
- }
-
-private:
- const ui64 OutputIndex;
- const TDqSolomonWriteParams WriteParams;
- const TString Url;
- NYql::NDq::IDqSinkActor::ICallbacks* const Callbacks;
- TDqSolomonWriteActorMetrics Metrics;
- i64 FreeSpace = 0;
- TActorId HttpProxyId;
-
- TString SourceId;
- bool ShouldNotifyNewFreeSpace = false;
- std::optional<NDqProto::TCheckpoint> CheckpointInProgress = std::nullopt;
- std::queue<std::variant<TMetricsToSend, NDqProto::TCheckpoint>> SendingBuffer;
- THashMap<NHttp::THttpOutgoingRequestPtr, TMetricsInflight> InflightBuffer;
-
- TMetricsEncoder UserMetricsEncoder;
+ CheckpointInProgress = std::nullopt;
+ }
+
+private:
+ const ui64 OutputIndex;
+ const TDqSolomonWriteParams WriteParams;
+ const TString Url;
+ NYql::NDq::IDqSinkActor::ICallbacks* const Callbacks;
+ TDqSolomonWriteActorMetrics Metrics;
+ i64 FreeSpace = 0;
+ TActorId HttpProxyId;
+
+ TString SourceId;
+ bool ShouldNotifyNewFreeSpace = false;
+ std::optional<NDqProto::TCheckpoint> CheckpointInProgress = std::nullopt;
+ std::queue<std::variant<TMetricsToSend, NDqProto::TCheckpoint>> SendingBuffer;
+ THashMap<NHttp::THttpOutgoingRequestPtr, TMetricsInflight> InflightBuffer;
+
+ TMetricsEncoder UserMetricsEncoder;
std::shared_ptr<NYdb::ICredentialsProvider> CredentialsProvider;
-};
-
-std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor(
- NYql::NSo::NProto::TDqSolomonShard&& settings,
- ui64 outputIndex,
- const THashMap<TString, TString>& secureParams,
- NYql::NDq::IDqSinkActor::ICallbacks* callbacks,
+};
+
+std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor(
+ NYql::NSo::NProto::TDqSolomonShard&& settings,
+ ui64 outputIndex,
+ const THashMap<TString, TString>& secureParams,
+ NYql::NDq::IDqSinkActor::ICallbacks* callbacks,
const NMonitoring::TDynamicCounterPtr& counters,
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
- i64 freeSpace)
-{
+ i64 freeSpace)
+{
const TString& tokenName = settings.GetToken().GetName();
const TString token = secureParams.Value(tokenName, TString());
-
- TDqSolomonWriteParams params {
- .Shard = std::move(settings),
- };
-
+
+ TDqSolomonWriteParams params {
+ .Shard = std::move(settings),
+ };
+
auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(credentialsFactory, token);
auto credentialsProvider = credentialsProviderFactory->CreateProvider();
- TDqSolomonWriteActor* actor = new TDqSolomonWriteActor(
- outputIndex,
- std::move(params),
- callbacks,
- counters,
+ TDqSolomonWriteActor* actor = new TDqSolomonWriteActor(
+ outputIndex,
+ std::move(params),
+ callbacks,
+ counters,
credentialsProvider,
- freeSpace);
- return {actor, actor};
-}
-
+ freeSpace);
+ return {actor, actor};
+}
+
void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) {
- factory.Register<NSo::NProto::TDqSolomonShard>("SolomonSink",
+ factory.Register<NSo::NProto::TDqSolomonShard>("SolomonSink",
[credentialsFactory](
- NYql::NSo::NProto::TDqSolomonShard&& settings,
- IDqSinkActorFactory::TArguments&& args)
- {
- auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
-
- return CreateDqSolomonWriteActor(
- std::move(settings),
- args.OutputIndex,
- args.SecureParams,
- args.Callback,
+ NYql::NSo::NProto::TDqSolomonShard&& settings,
+ IDqSinkActorFactory::TArguments&& args)
+ {
+ auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+
+ return CreateDqSolomonWriteActor(
+ std::move(settings),
+ args.OutputIndex,
+ args.SecureParams,
+ args.Callback,
counters,
credentialsFactory);
- });
-}
-
-} // namespace NYql::NDq
+ });
+}
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h
index 46810fca144..56c34cbb25f 100644
--- a/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h
+++ b/ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h
@@ -1,32 +1,32 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/utils/actors/http_sender.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h>
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h>
#include <ydb/library/yql/providers/common/token_accessor/client/factory.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/providers/solomon/proto/dq_solomon_shard.pb.h>
-
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/http/http_proxy.h>
-
-#include <util/generic/size_literals.h>
-#include <util/system/types.h>
-
-namespace NYql::NDq {
-
-constexpr i64 DqSolomonDefaultFreeSpace = 16_MB;
-
-std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor(
- NYql::NSo::NProto::TDqSolomonShard&& settings,
- ui64 outputIndex,
- const THashMap<TString, TString>& secureParams,
- NYql::NDq::IDqSinkActor::ICallbacks* callbacks,
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/actors/http/http_proxy.h>
+
+#include <util/generic/size_literals.h>
+#include <util/system/types.h>
+
+namespace NYql::NDq {
+
+constexpr i64 DqSolomonDefaultFreeSpace = 16_MB;
+
+std::pair<NYql::NDq::IDqSinkActor*, NActors::IActor*> CreateDqSolomonWriteActor(
+ NYql::NSo::NProto::TDqSolomonShard&& settings,
+ ui64 outputIndex,
+ const THashMap<TString, TString>& secureParams,
+ NYql::NDq::IDqSinkActor::ICallbacks* callbacks,
const NMonitoring::TDynamicCounterPtr& counters,
ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory,
- i64 freeSpace = DqSolomonDefaultFreeSpace);
-
+ i64 freeSpace = DqSolomonDefaultFreeSpace);
+
void RegisterDQSolomonWriteActorFactory(TDqSinkFactory& factory, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory);
-
-} // namespace NYql::NDq
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp b/ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp
index f3e7e571771..80637642fb3 100644
--- a/ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp
+++ b/ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp
@@ -1,149 +1,149 @@
-#include "metrics_encoder.h"
-
+#include "metrics_encoder.h"
+
#include <ydb/library/yql/public/udf/udf_data_type.h>
-
-#include <util/datetime/base.h>
-
-using namespace NYql;
-
-namespace {
-
-void BeginMetric(
- const NMonitoring::IMetricEncoderPtr& encoder,
- const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme)
-{
- switch (scheme.GetDataTypeId()) {
- case NUdf::TDataType<i8>::Id:
- case NUdf::TDataType<ui8>::Id:
- case NUdf::TDataType<i16>::Id:
- case NUdf::TDataType<ui16>::Id:
- case NUdf::TDataType<i32>::Id:
- case NUdf::TDataType<ui32>::Id:
- case NUdf::TDataType<i64>::Id:
- case NUdf::TDataType<ui64>::Id:
- encoder->OnMetricBegin(NMonitoring::EMetricType::IGAUGE);
- break;
- case NUdf::TDataType<float>::Id:
- case NUdf::TDataType<double>::Id:
- encoder->OnMetricBegin(NMonitoring::EMetricType::GAUGE);
- break;
- default:
- Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId());
- }
-}
-
-TInstant ParseTimestamp(
- const NUdf::TUnboxedValue& unboxedValue,
- const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme)
-{
- const auto& timestampValue = unboxedValue.GetElement(scheme.GetIndex());
- switch (scheme.GetDataTypeId()) {
- case NUdf::TDataType<NUdf::TDate>::Id:
- case NUdf::TDataType<NUdf::TTzDate>::Id:
- case NUdf::TDataType<NUdf::TDatetime>::Id:
- case NUdf::TDataType<NUdf::TTzDatetime>::Id:
- return TInstant::Seconds(timestampValue.Get<ui32>());
- case NUdf::TDataType<NUdf::TTimestamp>::Id:
- case NUdf::TDataType<NUdf::TTzTimestamp>::Id:
- return TInstant::MicroSeconds(timestampValue.Get<ui64>());
- default:
- Y_ENSURE(false, "Bad type for timestamp " << scheme.GetDataTypeId());
- }
-}
-
-void EncodeSensorValue(
- const NMonitoring::IMetricEncoderPtr& encoder,
- const TInstant& timestamp,
- const NUdf::TUnboxedValue& unboxedValue,
- const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme)
-{
- switch (scheme.GetDataTypeId()) {
- case NUdf::TDataType<i8>::Id:
- encoder->OnInt64(timestamp, unboxedValue.Get<i8>());
- break;
- case NUdf::TDataType<ui8>::Id:
- encoder->OnUint64(timestamp, unboxedValue.Get<ui8>());
- break;
- case NUdf::TDataType<i16>::Id:
- encoder->OnInt64(timestamp, unboxedValue.Get<i16>());
- break;
- case NUdf::TDataType<ui16>::Id:
- encoder->OnUint64(timestamp, unboxedValue.Get<ui16>());
- break;
- case NUdf::TDataType<i32>::Id:
- encoder->OnInt64(timestamp, unboxedValue.Get<i32>());
- break;
- case NUdf::TDataType<ui32>::Id:
- encoder->OnUint64(timestamp, unboxedValue.Get<ui32>());
- break;
- case NUdf::TDataType<i64>::Id:
- encoder->OnInt64(timestamp, unboxedValue.Get<i64>());
- break;
- case NUdf::TDataType<ui64>::Id:
- encoder->OnUint64(timestamp, unboxedValue.Get<ui64>());
- break;
- case NUdf::TDataType<float>::Id:
- encoder->OnDouble(timestamp, unboxedValue.Get<float>());
- break;
- case NUdf::TDataType<double>::Id:
- encoder->OnDouble(timestamp, unboxedValue.Get<double>());
- break;
- default:
- Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId());
- }
-}
-}
-
-namespace NYql::NDq {
-
-TMetricsEncoder::TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool useCloudFormat)
- : Scheme(scheme)
- , UseCloudFormat(useCloudFormat)
-{
- BeginNew();
-}
-
-void TMetricsEncoder::BeginNew() {
- Data.clear();
- DataOut.emplace(Data);
- SolomonEncoder = UseCloudFormat
- ? NMonitoring::BufferedEncoderCloudJson(&DataOut.value(), 0, "name")
- : NMonitoring::BufferedEncoderJson(&DataOut.value(), 0);
-
- SolomonEncoder->OnStreamBegin();
-}
-
-ui64 TMetricsEncoder::Append(const NUdf::TUnboxedValue& value)
-{
- TInstant timestamp = ParseTimestamp(value, Scheme.GetTimestamp());
-
- for (const auto& sensor : Scheme.GetSensors()) {
- BeginMetric(SolomonEncoder, sensor);
-
- SolomonEncoder->OnLabelsBegin();
- SolomonEncoder->OnLabel("name", sensor.GetKey());
- for (const auto& label : Scheme.GetLabels()) {
- const NUdf::TUnboxedValue& labelValue = value.GetElement(label.GetIndex());
- SolomonEncoder->OnLabel(label.GetKey(), TString(labelValue.AsStringRef()));
- }
- SolomonEncoder->OnLabelsEnd();
-
- const auto& sensorValue = value.GetElement(sensor.GetIndex());
- EncodeSensorValue(SolomonEncoder, timestamp, sensorValue, sensor);
-
- SolomonEncoder->OnMetricEnd();
- }
-
- return Scheme.GetSensors().size();
-}
-
-TString TMetricsEncoder::Encode() {
- SolomonEncoder->OnStreamEnd();
- SolomonEncoder->Close();
-
- TString res = Data;
- BeginNew();
- return res;
-}
-
-}
+
+#include <util/datetime/base.h>
+
+using namespace NYql;
+
+namespace {
+
+void BeginMetric(
+ const NMonitoring::IMetricEncoderPtr& encoder,
+ const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme)
+{
+ switch (scheme.GetDataTypeId()) {
+ case NUdf::TDataType<i8>::Id:
+ case NUdf::TDataType<ui8>::Id:
+ case NUdf::TDataType<i16>::Id:
+ case NUdf::TDataType<ui16>::Id:
+ case NUdf::TDataType<i32>::Id:
+ case NUdf::TDataType<ui32>::Id:
+ case NUdf::TDataType<i64>::Id:
+ case NUdf::TDataType<ui64>::Id:
+ encoder->OnMetricBegin(NMonitoring::EMetricType::IGAUGE);
+ break;
+ case NUdf::TDataType<float>::Id:
+ case NUdf::TDataType<double>::Id:
+ encoder->OnMetricBegin(NMonitoring::EMetricType::GAUGE);
+ break;
+ default:
+ Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId());
+ }
+}
+
+TInstant ParseTimestamp(
+ const NUdf::TUnboxedValue& unboxedValue,
+ const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme)
+{
+ const auto& timestampValue = unboxedValue.GetElement(scheme.GetIndex());
+ switch (scheme.GetDataTypeId()) {
+ case NUdf::TDataType<NUdf::TDate>::Id:
+ case NUdf::TDataType<NUdf::TTzDate>::Id:
+ case NUdf::TDataType<NUdf::TDatetime>::Id:
+ case NUdf::TDataType<NUdf::TTzDatetime>::Id:
+ return TInstant::Seconds(timestampValue.Get<ui32>());
+ case NUdf::TDataType<NUdf::TTimestamp>::Id:
+ case NUdf::TDataType<NUdf::TTzTimestamp>::Id:
+ return TInstant::MicroSeconds(timestampValue.Get<ui64>());
+ default:
+ Y_ENSURE(false, "Bad type for timestamp " << scheme.GetDataTypeId());
+ }
+}
+
+void EncodeSensorValue(
+ const NMonitoring::IMetricEncoderPtr& encoder,
+ const TInstant& timestamp,
+ const NUdf::TUnboxedValue& unboxedValue,
+ const NYql::NSo::NProto::TDqSolomonSchemeItem& scheme)
+{
+ switch (scheme.GetDataTypeId()) {
+ case NUdf::TDataType<i8>::Id:
+ encoder->OnInt64(timestamp, unboxedValue.Get<i8>());
+ break;
+ case NUdf::TDataType<ui8>::Id:
+ encoder->OnUint64(timestamp, unboxedValue.Get<ui8>());
+ break;
+ case NUdf::TDataType<i16>::Id:
+ encoder->OnInt64(timestamp, unboxedValue.Get<i16>());
+ break;
+ case NUdf::TDataType<ui16>::Id:
+ encoder->OnUint64(timestamp, unboxedValue.Get<ui16>());
+ break;
+ case NUdf::TDataType<i32>::Id:
+ encoder->OnInt64(timestamp, unboxedValue.Get<i32>());
+ break;
+ case NUdf::TDataType<ui32>::Id:
+ encoder->OnUint64(timestamp, unboxedValue.Get<ui32>());
+ break;
+ case NUdf::TDataType<i64>::Id:
+ encoder->OnInt64(timestamp, unboxedValue.Get<i64>());
+ break;
+ case NUdf::TDataType<ui64>::Id:
+ encoder->OnUint64(timestamp, unboxedValue.Get<ui64>());
+ break;
+ case NUdf::TDataType<float>::Id:
+ encoder->OnDouble(timestamp, unboxedValue.Get<float>());
+ break;
+ case NUdf::TDataType<double>::Id:
+ encoder->OnDouble(timestamp, unboxedValue.Get<double>());
+ break;
+ default:
+ Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId());
+ }
+}
+}
+
+namespace NYql::NDq {
+
+TMetricsEncoder::TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool useCloudFormat)
+ : Scheme(scheme)
+ , UseCloudFormat(useCloudFormat)
+{
+ BeginNew();
+}
+
+void TMetricsEncoder::BeginNew() {
+ Data.clear();
+ DataOut.emplace(Data);
+ SolomonEncoder = UseCloudFormat
+ ? NMonitoring::BufferedEncoderCloudJson(&DataOut.value(), 0, "name")
+ : NMonitoring::BufferedEncoderJson(&DataOut.value(), 0);
+
+ SolomonEncoder->OnStreamBegin();
+}
+
+ui64 TMetricsEncoder::Append(const NUdf::TUnboxedValue& value)
+{
+ TInstant timestamp = ParseTimestamp(value, Scheme.GetTimestamp());
+
+ for (const auto& sensor : Scheme.GetSensors()) {
+ BeginMetric(SolomonEncoder, sensor);
+
+ SolomonEncoder->OnLabelsBegin();
+ SolomonEncoder->OnLabel("name", sensor.GetKey());
+ for (const auto& label : Scheme.GetLabels()) {
+ const NUdf::TUnboxedValue& labelValue = value.GetElement(label.GetIndex());
+ SolomonEncoder->OnLabel(label.GetKey(), TString(labelValue.AsStringRef()));
+ }
+ SolomonEncoder->OnLabelsEnd();
+
+ const auto& sensorValue = value.GetElement(sensor.GetIndex());
+ EncodeSensorValue(SolomonEncoder, timestamp, sensorValue, sensor);
+
+ SolomonEncoder->OnMetricEnd();
+ }
+
+ return Scheme.GetSensors().size();
+}
+
+TString TMetricsEncoder::Encode() {
+ SolomonEncoder->OnStreamEnd();
+ SolomonEncoder->Close();
+
+ TString res = Data;
+ BeginNew();
+ return res;
+}
+
+}
diff --git a/ydb/library/yql/providers/solomon/async_io/metrics_encoder.h b/ydb/library/yql/providers/solomon/async_io/metrics_encoder.h
index 2826a3ca4ac..b759e288e46 100644
--- a/ydb/library/yql/providers/solomon/async_io/metrics_encoder.h
+++ b/ydb/library/yql/providers/solomon/async_io/metrics_encoder.h
@@ -1,31 +1,31 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/utils/actors/http_sender.h>
-
+
#include <ydb/library/yql/public/udf/udf_value.h>
-
-#include <library/cpp/monlib/encode/json/json.h>
-
-#include <util/stream/str.h>
-
-
-namespace NYql::NDq {
-
-class TMetricsEncoder {
-public:
- explicit TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool cloudFormat);
-
- void BeginNew();
- ui64 Append(const NUdf::TUnboxedValue& value);
- TString Encode();
-
-private:
- const NSo::NProto::TDqSolomonShardScheme& Scheme;
-
- TString Data;
- std::optional<TStringOutput> DataOut;
- NMonitoring::IMetricEncoderPtr SolomonEncoder;
- bool UseCloudFormat;
-};
-
-}
+
+#include <library/cpp/monlib/encode/json/json.h>
+
+#include <util/stream/str.h>
+
+
+namespace NYql::NDq {
+
+class TMetricsEncoder {
+public:
+ explicit TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool cloudFormat);
+
+ void BeginNew();
+ ui64 Append(const NUdf::TUnboxedValue& value);
+ TString Encode();
+
+private:
+ const NSo::NProto::TDqSolomonShardScheme& Scheme;
+
+ TString Data;
+ std::optional<TStringOutput> DataOut;
+ NMonitoring::IMetricEncoderPtr SolomonEncoder;
+ bool UseCloudFormat;
+};
+
+}
diff --git a/ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp b/ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp
index d9de9f17ed6..3fca3a2bdc5 100644
--- a/ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp
+++ b/ydb/library/yql/providers/solomon/async_io/ut/dq_solomon_write_actor_ut.cpp
@@ -1,184 +1,184 @@
-#include "ut_helpers.h"
-
+#include "ut_helpers.h"
+
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <thread>
-
-namespace NYql::NDq {
-
-using namespace NKikimr::NMiniKQL;
-
-constexpr TDuration WaitTimeout = TDuration::Seconds(10);
-
-namespace {
- void TestWriteBigBatch(bool isCloud) {
- const int batchSize = 7500;
- CleanupSolomon("cloudId1", "folderId1", "custom", isCloud);
-
- TFakeCASetup setup;
- InitSink(setup, BuildSolomonShardSettings(isCloud));
-
- auto issue = setup.SinkPromises.Issue.GetFuture();
- setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
- TUnboxedValueVector res;
- res.reserve(batchSize);
-
- for (int i = 0; i < batchSize; i++) {
- res.emplace_back(CreateStruct(holderFactory, {
- NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)),
- NKikimr::NMiniKQL::MakeString(std::to_string(i)),
- NUdf::TUnboxedValuePod(678)
- }));
- }
-
- return res;
- });
- UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString());
-
- const auto metrics = GetSolomonMetrics("folderId1", "custom");
- UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize);
- }
-}
-
-Y_UNIT_TEST_SUITE(TDqSolomonWriteActorTest) {
- Y_UNIT_TEST(TestWriteFormat) {
- CleanupSolomon("cloudId1", "folderId1", "custom", true);
-
- TFakeCASetup setup;
- InitSink(setup, BuildSolomonShardSettings(true));
-
- auto issue = setup.SinkPromises.Issue.GetFuture();
- setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
- NUdf::TUnboxedValue val1 = CreateStruct(holderFactory, {
- NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(1624811684)),
- NKikimr::NMiniKQL::MakeString("123"),
- NUdf::TUnboxedValuePod(678)
- });
- return TUnboxedValueVector { std::move(val1) };
- });
- UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString());
-
- const auto metrics = GetSolomonMetrics("folderId1", "custom");
- const auto expected = R"([
- {
- "labels": [
- [
- "label1",
- "123"
- ],
- [
- "name",
- "sensor1"
- ]
- ],
- "ts": "1970-01-01T00:27:04.811684Z",
- "value": 678
- }
-])";
- UNIT_ASSERT_STRINGS_EQUAL(metrics, expected);
- }
-
- Y_UNIT_TEST(TestWriteBigBatchMonitoring) {
- TestWriteBigBatch(true);
- }
-
- Y_UNIT_TEST(TestWriteBigBatchSolomon) {
- //TestWriteBigBatch(false);
- }
-
- Y_UNIT_TEST(TestWriteWithTimeseries) {
- const int batchSize = 10;
- CleanupSolomon("cloudId1", "folderId1", "custom", true);
-
- TFakeCASetup setup;
- InitSink(setup, BuildSolomonShardSettings(true));
-
- auto issue = setup.SinkPromises.Issue.GetFuture();
- setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
- TUnboxedValueVector res;
- res.reserve(batchSize);
-
- for (int i = 0; i < batchSize; i++) {
- res.emplace_back(CreateStruct(holderFactory, {
- NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)),
- NKikimr::NMiniKQL::MakeString("123"),
- NUdf::TUnboxedValuePod(678)
- }));
- }
-
- return res;
- });
- UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString());
-
- const auto metrics = GetSolomonMetrics("folderId1", "custom");
- UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize);
- }
-
- Y_UNIT_TEST(TestCheckpoints) {
- const int batchSize = 2400;
-
- {
- TFakeCASetup setup;
- CleanupSolomon("cloudId1", "folderId1", "custom", true);
- InitSink(setup, BuildSolomonShardSettings(true));
-
- auto stateSaved = setup.SinkPromises.StateSaved.GetFuture();
- setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
- TUnboxedValueVector res;
- res.reserve(batchSize);
-
- for (int i = 0; i < batchSize; i++) {
- res.emplace_back(CreateStruct(holderFactory, {
- NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)),
- NKikimr::NMiniKQL::MakeString(std::to_string(i)),
- NUdf::TUnboxedValuePod(678)
- }));
- }
-
- return res;
- }, CreateCheckpoint(1));
- UNIT_ASSERT(stateSaved.Wait(WaitTimeout));
-
- const auto metrics = GetSolomonMetrics("folderId1", "custom");
- UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize);
- }
- }
-
- Y_UNIT_TEST(TestShouldReturnAfterCheckpoint) {
- {
- TFakeCASetup setup;
- CleanupSolomon("cloudId1", "folderId1", "custom", true);
- InitSink(setup, BuildSolomonShardSettings(true));
-
- auto stateSaved = setup.SinkPromises.StateSaved.GetFuture();
- setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
- return TUnboxedValueVector {
- CreateStruct(holderFactory, {
- NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(200000)),
- NKikimr::NMiniKQL::MakeString("abc"),
- NUdf::TUnboxedValuePod(678)
- })};
- }, CreateCheckpoint(1));
- UNIT_ASSERT(stateSaved.Wait(WaitTimeout));
-
- auto issue = setup.SinkPromises.Issue.GetFuture();
- setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
- return TUnboxedValueVector {
- CreateStruct(holderFactory, {
- NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(200001)),
- NKikimr::NMiniKQL::MakeString("cba"),
- NUdf::TUnboxedValuePod(678)
- })};
- });
- UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString());
-
- const auto metrics = GetSolomonMetrics("folderId1", "custom");
- UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), 2);
- }
- }
-}
-
-} // namespace NKikimr::NMiniKQL
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <thread>
+
+namespace NYql::NDq {
+
+using namespace NKikimr::NMiniKQL;
+
+constexpr TDuration WaitTimeout = TDuration::Seconds(10);
+
+namespace {
+ void TestWriteBigBatch(bool isCloud) {
+ const int batchSize = 7500;
+ CleanupSolomon("cloudId1", "folderId1", "custom", isCloud);
+
+ TFakeCASetup setup;
+ InitSink(setup, BuildSolomonShardSettings(isCloud));
+
+ auto issue = setup.SinkPromises.Issue.GetFuture();
+ setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
+ TUnboxedValueVector res;
+ res.reserve(batchSize);
+
+ for (int i = 0; i < batchSize; i++) {
+ res.emplace_back(CreateStruct(holderFactory, {
+ NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)),
+ NKikimr::NMiniKQL::MakeString(std::to_string(i)),
+ NUdf::TUnboxedValuePod(678)
+ }));
+ }
+
+ return res;
+ });
+ UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString());
+
+ const auto metrics = GetSolomonMetrics("folderId1", "custom");
+ UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize);
+ }
+}
+
+Y_UNIT_TEST_SUITE(TDqSolomonWriteActorTest) {
+ Y_UNIT_TEST(TestWriteFormat) {
+ CleanupSolomon("cloudId1", "folderId1", "custom", true);
+
+ TFakeCASetup setup;
+ InitSink(setup, BuildSolomonShardSettings(true));
+
+ auto issue = setup.SinkPromises.Issue.GetFuture();
+ setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
+ NUdf::TUnboxedValue val1 = CreateStruct(holderFactory, {
+ NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(1624811684)),
+ NKikimr::NMiniKQL::MakeString("123"),
+ NUdf::TUnboxedValuePod(678)
+ });
+ return TUnboxedValueVector { std::move(val1) };
+ });
+ UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString());
+
+ const auto metrics = GetSolomonMetrics("folderId1", "custom");
+ const auto expected = R"([
+ {
+ "labels": [
+ [
+ "label1",
+ "123"
+ ],
+ [
+ "name",
+ "sensor1"
+ ]
+ ],
+ "ts": "1970-01-01T00:27:04.811684Z",
+ "value": 678
+ }
+])";
+ UNIT_ASSERT_STRINGS_EQUAL(metrics, expected);
+ }
+
+ Y_UNIT_TEST(TestWriteBigBatchMonitoring) {
+ TestWriteBigBatch(true);
+ }
+
+ Y_UNIT_TEST(TestWriteBigBatchSolomon) {
+ //TestWriteBigBatch(false);
+ }
+
+ Y_UNIT_TEST(TestWriteWithTimeseries) {
+ const int batchSize = 10;
+ CleanupSolomon("cloudId1", "folderId1", "custom", true);
+
+ TFakeCASetup setup;
+ InitSink(setup, BuildSolomonShardSettings(true));
+
+ auto issue = setup.SinkPromises.Issue.GetFuture();
+ setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
+ TUnboxedValueVector res;
+ res.reserve(batchSize);
+
+ for (int i = 0; i < batchSize; i++) {
+ res.emplace_back(CreateStruct(holderFactory, {
+ NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)),
+ NKikimr::NMiniKQL::MakeString("123"),
+ NUdf::TUnboxedValuePod(678)
+ }));
+ }
+
+ return res;
+ });
+ UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString());
+
+ const auto metrics = GetSolomonMetrics("folderId1", "custom");
+ UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize);
+ }
+
+ Y_UNIT_TEST(TestCheckpoints) {
+ const int batchSize = 2400;
+
+ {
+ TFakeCASetup setup;
+ CleanupSolomon("cloudId1", "folderId1", "custom", true);
+ InitSink(setup, BuildSolomonShardSettings(true));
+
+ auto stateSaved = setup.SinkPromises.StateSaved.GetFuture();
+ setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
+ TUnboxedValueVector res;
+ res.reserve(batchSize);
+
+ for (int i = 0; i < batchSize; i++) {
+ res.emplace_back(CreateStruct(holderFactory, {
+ NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(i + 200000)),
+ NKikimr::NMiniKQL::MakeString(std::to_string(i)),
+ NUdf::TUnboxedValuePod(678)
+ }));
+ }
+
+ return res;
+ }, CreateCheckpoint(1));
+ UNIT_ASSERT(stateSaved.Wait(WaitTimeout));
+
+ const auto metrics = GetSolomonMetrics("folderId1", "custom");
+ UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), batchSize);
+ }
+ }
+
+ Y_UNIT_TEST(TestShouldReturnAfterCheckpoint) {
+ {
+ TFakeCASetup setup;
+ CleanupSolomon("cloudId1", "folderId1", "custom", true);
+ InitSink(setup, BuildSolomonShardSettings(true));
+
+ auto stateSaved = setup.SinkPromises.StateSaved.GetFuture();
+ setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
+ return TUnboxedValueVector {
+ CreateStruct(holderFactory, {
+ NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(200000)),
+ NKikimr::NMiniKQL::MakeString("abc"),
+ NUdf::TUnboxedValuePod(678)
+ })};
+ }, CreateCheckpoint(1));
+ UNIT_ASSERT(stateSaved.Wait(WaitTimeout));
+
+ auto issue = setup.SinkPromises.Issue.GetFuture();
+ setup.SinkWrite([](NKikimr::NMiniKQL::THolderFactory& holderFactory){
+ return TUnboxedValueVector {
+ CreateStruct(holderFactory, {
+ NUdf::TUnboxedValuePod(static_cast<NUdf::TDataType<NUdf::TTimestamp>::TLayout>(200001)),
+ NKikimr::NMiniKQL::MakeString("cba"),
+ NUdf::TUnboxedValuePod(678)
+ })};
+ });
+ UNIT_ASSERT_C(!issue.Wait(WaitTimeout), issue.GetValue().ToString());
+
+ const auto metrics = GetSolomonMetrics("folderId1", "custom");
+ UNIT_ASSERT_EQUAL(GetMetricsCount(metrics), 2);
+ }
+ }
+}
+
+} // namespace NKikimr::NMiniKQL
diff --git a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp
index cf9cffdfa77..a97b2af7801 100644
--- a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp
+++ b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.cpp
@@ -1,126 +1,126 @@
-#include "ut_helpers.h"
-
+#include "ut_helpers.h"
+
#include <ydb/core/testlib/basics/appdata.h>
-
-#include <library/cpp/http/simple/http_client.h>
-#include <library/cpp/json/json_reader.h>
-#include <library/cpp/retry/retry.h>
-
-#include <util/system/guard.h>
-
-
-namespace NYql::NDq {
-
-using namespace NKikimr::NMiniKQL;
-
-namespace {
-
-void FillDqSolomonScheme(NSo::NProto::TDqSolomonShardScheme& scheme) {
- scheme.MutableTimestamp()->SetKey("ts");
- scheme.MutableTimestamp()->SetIndex(0);
- scheme.MutableTimestamp()->SetDataTypeId(NUdf::TDataType<NUdf::TTimestamp>::Id);
-
- NSo::NProto::TDqSolomonSchemeItem label;
- label.SetKey("label1");
- label.SetIndex(1);
- label.SetDataTypeId(NUdf::TDataType<ui32>::Id);
-
- NSo::NProto::TDqSolomonSchemeItem sensor;
- sensor.SetKey("sensor1");
- sensor.SetIndex(2);
- sensor.SetDataTypeId(NUdf::TDataType<ui32>::Id);
-
- scheme.MutableLabels()->Add(std::move(label));
- scheme.MutableSensors()->Add(std::move(sensor));
-}
-
-}
-
-void InitSink(
- TFakeCASetup& caSetup,
- NSo::NProto::TDqSolomonShard&& settings,
- i64 freeSpace)
-{
- auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
- const THashMap<TString, TString> secureParams;
-
- caSetup.Execute([&](TFakeActor& actor) {
- auto [dqSink, dqSinkAsActor] = CreateDqSolomonWriteActor(
- std::move(settings),
- 0,
- secureParams,
- &actor.GetSinkCallbacks(),
- counters,
+
+#include <library/cpp/http/simple/http_client.h>
+#include <library/cpp/json/json_reader.h>
+#include <library/cpp/retry/retry.h>
+
+#include <util/system/guard.h>
+
+
+namespace NYql::NDq {
+
+using namespace NKikimr::NMiniKQL;
+
+namespace {
+
+void FillDqSolomonScheme(NSo::NProto::TDqSolomonShardScheme& scheme) {
+ scheme.MutableTimestamp()->SetKey("ts");
+ scheme.MutableTimestamp()->SetIndex(0);
+ scheme.MutableTimestamp()->SetDataTypeId(NUdf::TDataType<NUdf::TTimestamp>::Id);
+
+ NSo::NProto::TDqSolomonSchemeItem label;
+ label.SetKey("label1");
+ label.SetIndex(1);
+ label.SetDataTypeId(NUdf::TDataType<ui32>::Id);
+
+ NSo::NProto::TDqSolomonSchemeItem sensor;
+ sensor.SetKey("sensor1");
+ sensor.SetIndex(2);
+ sensor.SetDataTypeId(NUdf::TDataType<ui32>::Id);
+
+ scheme.MutableLabels()->Add(std::move(label));
+ scheme.MutableSensors()->Add(std::move(sensor));
+}
+
+}
+
+void InitSink(
+ TFakeCASetup& caSetup,
+ NSo::NProto::TDqSolomonShard&& settings,
+ i64 freeSpace)
+{
+ auto counters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ const THashMap<TString, TString> secureParams;
+
+ caSetup.Execute([&](TFakeActor& actor) {
+ auto [dqSink, dqSinkAsActor] = CreateDqSolomonWriteActor(
+ std::move(settings),
+ 0,
+ secureParams,
+ &actor.GetSinkCallbacks(),
+ counters,
nullptr,
- freeSpace);
-
- actor.InitSink(dqSink, dqSinkAsActor);
- });
-}
-
-void CleanupSolomon(TString cloudId, TString folderId, TString service, bool isCloud) {
- const auto solomonPort = TString(getenv("SOLOMON_PORT"));
- TSimpleHttpClient httpClient("localhost", std::stoi(solomonPort));
- TStringStream str;
- TStringBuilder builder;
- builder << "/cleanup";
- if (isCloud) {
- builder << "?folderId=" << folderId << "&service=" << service;
- } else {
- builder << "?project=" << cloudId << "&cluster=" << folderId << "&service=" << service;
- }
-
- DoWithRetry(
- [&]{ httpClient.DoPost(builder, "", &str); },
- TRetryOptions(3, TDuration::Seconds(1)),
- true);
-}
-
-TString GetSolomonMetrics(TString folderId, TString service) {
- const auto solomonPort = TString(getenv("SOLOMON_PORT"));
- TSimpleHttpClient httpClient("localhost", std::stoi(solomonPort));
- TStringStream str;
- httpClient.DoGet("/metrics?folderId=" + folderId + "&service=" + service, &str);
- return TString(str.Str());
-}
-
-NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud) {
- NSo::NProto::TDqSolomonShard settings;
- settings.SetEndpoint(TStringBuilder() << getenv("SOLOMON_HOST") << ":" << TString(getenv("SOLOMON_PORT")));
- if (isCloud) {
- settings.SetProject("folderId1");
- settings.SetCluster("folderId1");
- settings.SetService("custom");
- } else {
- settings.SetProject("cloudId1");
- settings.SetCluster("folderId1");
- settings.SetService("custom");
- }
-
- settings.SetClusterType(isCloud ? NSo::NProto::ESolomonClusterType::CT_MONITORING : NSo::NProto::ESolomonClusterType::CT_SOLOMON);
+ freeSpace);
+
+ actor.InitSink(dqSink, dqSinkAsActor);
+ });
+}
+
+void CleanupSolomon(TString cloudId, TString folderId, TString service, bool isCloud) {
+ const auto solomonPort = TString(getenv("SOLOMON_PORT"));
+ TSimpleHttpClient httpClient("localhost", std::stoi(solomonPort));
+ TStringStream str;
+ TStringBuilder builder;
+ builder << "/cleanup";
+ if (isCloud) {
+ builder << "?folderId=" << folderId << "&service=" << service;
+ } else {
+ builder << "?project=" << cloudId << "&cluster=" << folderId << "&service=" << service;
+ }
+
+ DoWithRetry(
+ [&]{ httpClient.DoPost(builder, "", &str); },
+ TRetryOptions(3, TDuration::Seconds(1)),
+ true);
+}
+
+TString GetSolomonMetrics(TString folderId, TString service) {
+ const auto solomonPort = TString(getenv("SOLOMON_PORT"));
+ TSimpleHttpClient httpClient("localhost", std::stoi(solomonPort));
+ TStringStream str;
+ httpClient.DoGet("/metrics?folderId=" + folderId + "&service=" + service, &str);
+ return TString(str.Str());
+}
+
+NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud) {
+ NSo::NProto::TDqSolomonShard settings;
+ settings.SetEndpoint(TStringBuilder() << getenv("SOLOMON_HOST") << ":" << TString(getenv("SOLOMON_PORT")));
+ if (isCloud) {
+ settings.SetProject("folderId1");
+ settings.SetCluster("folderId1");
+ settings.SetService("custom");
+ } else {
+ settings.SetProject("cloudId1");
+ settings.SetCluster("folderId1");
+ settings.SetService("custom");
+ }
+
+ settings.SetClusterType(isCloud ? NSo::NProto::ESolomonClusterType::CT_MONITORING : NSo::NProto::ESolomonClusterType::CT_SOLOMON);
settings.SetUseSsl(false);
-
- FillDqSolomonScheme(*settings.MutableScheme());
-
- return settings;
-}
-
-NUdf::TUnboxedValue CreateStruct(
- NKikimr::NMiniKQL::THolderFactory& holderFactory,
- std::initializer_list<NUdf::TUnboxedValuePod> fields)
-{
- NUdf::TUnboxedValue* itemsPtr = nullptr;
- auto structValues = holderFactory.CreateDirectArrayHolder(fields.size(), itemsPtr);
- for (auto&& field : fields) {
- *(itemsPtr++) = std::move(field);
- }
- return structValues;
-}
-
-int GetMetricsCount(TString metrics) {
- NJson::TJsonValue json;
- NJson::ReadJsonTree(metrics, &json, true);
- return json.GetArray().size();
-}
-
-} // namespace NYql::NDq
+
+ FillDqSolomonScheme(*settings.MutableScheme());
+
+ return settings;
+}
+
+NUdf::TUnboxedValue CreateStruct(
+ NKikimr::NMiniKQL::THolderFactory& holderFactory,
+ std::initializer_list<NUdf::TUnboxedValuePod> fields)
+{
+ NUdf::TUnboxedValue* itemsPtr = nullptr;
+ auto structValues = holderFactory.CreateDirectArrayHolder(fields.size(), itemsPtr);
+ for (auto&& field : fields) {
+ *(itemsPtr++) = std::move(field);
+ }
+ return structValues;
+}
+
+int GetMetricsCount(TString metrics) {
+ NJson::TJsonValue json;
+ NJson::ReadJsonTree(metrics, &json, true);
+ return json.GetArray().size();
+}
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h
index 9b7b6b57e7c..5ba4931e60e 100644
--- a/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h
+++ b/ydb/library/yql/providers/solomon/async_io/ut/ut_helpers.h
@@ -7,31 +7,31 @@
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h>
#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h>
#include <ydb/library/yql/minikql/mkql_alloc.h>
-
+
#include <ydb/core/testlib/basics/runtime.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <chrono>
-#include <queue>
-
-namespace NYql::NDq {
-
-void InitSink(
- TFakeCASetup& caSetup,
- NSo::NProto::TDqSolomonShard&& settings,
- i64 freeSpace = 100000);
-
-void CleanupSolomon(TString cloudId, TString folderId, TString service, bool isCloud);
-
-TString GetSolomonMetrics(TString folderId, TString service);
-
-NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud);
-
-NUdf::TUnboxedValue CreateStruct(
- NKikimr::NMiniKQL::THolderFactory& holderFactory,
- std::initializer_list<NUdf::TUnboxedValuePod> fields);
-
-int GetMetricsCount(TString metrics);
-
-} // namespace NYql::NDq
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <chrono>
+#include <queue>
+
+namespace NYql::NDq {
+
+void InitSink(
+ TFakeCASetup& caSetup,
+ NSo::NProto::TDqSolomonShard&& settings,
+ i64 freeSpace = 100000);
+
+void CleanupSolomon(TString cloudId, TString folderId, TString service, bool isCloud);
+
+TString GetSolomonMetrics(TString folderId, TString service);
+
+NSo::NProto::TDqSolomonShard BuildSolomonShardSettings(bool isCloud);
+
+NUdf::TUnboxedValue CreateStruct(
+ NKikimr::NMiniKQL::THolderFactory& holderFactory,
+ std::initializer_list<NUdf::TUnboxedValuePod> fields);
+
+int GetMetricsCount(TString metrics);
+
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/solomon/async_io/ut/ya.make b/ydb/library/yql/providers/solomon/async_io/ut/ya.make
index a9aa1845ed7..110566a0027 100644
--- a/ydb/library/yql/providers/solomon/async_io/ut/ya.make
+++ b/ydb/library/yql/providers/solomon/async_io/ut/ya.make
@@ -1,21 +1,21 @@
UNITTEST_FOR(ydb/library/yql/providers/solomon/async_io)
-
-OWNER(
- d-mokhnatkin
+
+OWNER(
+ d-mokhnatkin
g:yq
- g:yql
-)
-
+ g:yql
+)
+
INCLUDE(${ARCADIA_ROOT}/kikimr/yq/tools/solomon_emulator/recipe/recipe.inc)
-
-SRCS(
- dq_solomon_write_actor_ut.cpp
- ut_helpers.cpp
-)
-
-PEERDIR(
- library/cpp/http/simple
- library/cpp/retry
+
+SRCS(
+ dq_solomon_write_actor_ut.cpp
+ ut_helpers.cpp
+)
+
+PEERDIR(
+ library/cpp/http/simple
+ library/cpp/retry
ydb/core/testlib/basics
ydb/library/yql/minikql
ydb/library/yql/minikql/computation
@@ -23,10 +23,10 @@ PEERDIR(
ydb/library/yql/sql
ydb/library/yql/providers/common/comp_nodes
ydb/library/yql/providers/common/ut_helpers
-)
-
-YQL_LAST_ABI_VERSION()
-
+)
+
+YQL_LAST_ABI_VERSION()
+
REQUIREMENTS(ram:12)
-END()
+END()
diff --git a/ydb/library/yql/providers/solomon/async_io/ya.make b/ydb/library/yql/providers/solomon/async_io/ya.make
index 4f1edaa5e3f..ea04c723e73 100644
--- a/ydb/library/yql/providers/solomon/async_io/ya.make
+++ b/ydb/library/yql/providers/solomon/async_io/ya.make
@@ -1,18 +1,18 @@
-LIBRARY()
-
-OWNER(
+LIBRARY()
+
+OWNER(
g:yq
- g:yql
-)
-
-SRCS(
- dq_solomon_write_actor.cpp
- metrics_encoder.cpp
-)
-
-PEERDIR(
- library/cpp/json/easy_parse
- library/cpp/monlib/encode/json
+ g:yql
+)
+
+SRCS(
+ dq_solomon_write_actor.cpp
+ metrics_encoder.cpp
+)
+
+PEERDIR(
+ library/cpp/json/easy_parse
+ library/cpp/monlib/encode/json
ydb/library/yql/minikql/computation
ydb/library/yql/providers/common/token_accessor/client
ydb/library/yql/public/types
@@ -20,12 +20,12 @@ PEERDIR(
ydb/library/yql/utils/log
ydb/library/yql/dq/actors/compute
ydb/library/yql/providers/solomon/proto
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
-
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
IF (NOT OPENSOURCE)
IF (OS_LINUX)
# Solomon recipe is supported only for linux.
@@ -33,5 +33,5 @@ IF (NOT OPENSOURCE)
ut
)
ENDIF()
-ENDIF()
+ENDIF()
diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json
index 1168b665bbb..d064daa1b40 100644
--- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json
+++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json
@@ -63,28 +63,28 @@
{"Index": 2, "Name": "Shard", "Type": "TCoAtom"},
{"Index": 3, "Name": "Input", "Type": "TExprBase"}
]
- },
- {
- "Name": "TSoShard",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "SoShard"},
- "Children": [
- {"Index": 0, "Name": "SolomonCluster", "Type": "TCoAtom"},
- {"Index": 1, "Name": "Project", "Type": "TCoAtom"},
- {"Index": 2, "Name": "Cluster", "Type": "TCoAtom"},
- {"Index": 3, "Name": "Service", "Type": "TCoAtom"},
+ },
+ {
+ "Name": "TSoShard",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "SoShard"},
+ "Children": [
+ {"Index": 0, "Name": "SolomonCluster", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "Project", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "Cluster", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "Service", "Type": "TCoAtom"},
{"Index": 4, "Name": "Token", "Type": "TCoSecureParam", "Optional": true}
- ]
- },
- {
- "Name": "TDqSoShardSink",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "DqSoShardSink"},
- "Children": [
- {"Index": 0, "Name": "Shard", "Type": "TSoShard"},
- {"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"},
- {"Index": 2, "Name": "Token", "Type": "TCoSecureParam", "Optional": true}
- ]
+ ]
+ },
+ {
+ "Name": "TDqSoShardSink",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "DqSoShardSink"},
+ "Children": [
+ {"Index": 0, "Name": "Shard", "Type": "TSoShard"},
+ {"Index": 1, "Name": "Settings", "Type": "TCoNameValueTupleList"},
+ {"Index": 2, "Name": "Token", "Type": "TCoSecureParam", "Optional": true}
+ ]
}
]
}
diff --git a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto
index bd1ac45bf25..3bb9d19d51d 100644
--- a/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto
+++ b/ydb/library/yql/providers/solomon/proto/dq_solomon_shard.proto
@@ -1,41 +1,41 @@
-syntax = "proto3";
-option cc_enable_arenas = true;
-
-package NYql.NSo.NProto;
-
-enum ESolomonClusterType {
- CT_UNSPECIFIED = 0;
- CT_SOLOMON = 1;
- CT_MONITORING = 2;
-}
-
+syntax = "proto3";
+option cc_enable_arenas = true;
+
+package NYql.NSo.NProto;
+
+enum ESolomonClusterType {
+ CT_UNSPECIFIED = 0;
+ CT_SOLOMON = 1;
+ CT_MONITORING = 2;
+}
+
message TToken {
- string Name = 1;
-}
-
-message TDqSolomonSchemeItem {
- string Key = 1;
- uint32 Index = 2;
- uint32 DataTypeId = 3;
-}
-
-message TDqSolomonShardScheme {
- TDqSolomonSchemeItem Timestamp = 1;
- repeated TDqSolomonSchemeItem Labels = 2;
- repeated TDqSolomonSchemeItem Sensors = 3;
-};
-
-message TDqSolomonShard {
- string Endpoint = 1;
- string Project = 2;
- string Cluster = 3;
- string Service = 4;
-
- ESolomonClusterType ClusterType = 20;
+ string Name = 1;
+}
+
+message TDqSolomonSchemeItem {
+ string Key = 1;
+ uint32 Index = 2;
+ uint32 DataTypeId = 3;
+}
+
+message TDqSolomonShardScheme {
+ TDqSolomonSchemeItem Timestamp = 1;
+ repeated TDqSolomonSchemeItem Labels = 2;
+ repeated TDqSolomonSchemeItem Sensors = 3;
+};
+
+message TDqSolomonShard {
+ string Endpoint = 1;
+ string Project = 2;
+ string Cluster = 3;
+ string Service = 4;
+
+ ESolomonClusterType ClusterType = 20;
bool UseSsl = 21;
-
- TDqSolomonShardScheme Scheme = 30;
-
- string ServiceAccount = 40;
+
+ TDqSolomonShardScheme Scheme = 30;
+
+ string ServiceAccount = 40;
TToken Token = 41;
-}
+}
diff --git a/ydb/library/yql/providers/solomon/proto/ya.make b/ydb/library/yql/providers/solomon/proto/ya.make
index 26362702196..b30b1d5cfd0 100644
--- a/ydb/library/yql/providers/solomon/proto/ya.make
+++ b/ydb/library/yql/providers/solomon/proto/ya.make
@@ -1,14 +1,14 @@
-PROTO_LIBRARY()
-
-OWNER(
+PROTO_LIBRARY()
+
+OWNER(
g:yq
- g:yql
-)
-
-SRCS(
- dq_solomon_shard.proto
-)
-
-EXCLUDE_TAGS(GO_PROTO)
-
-END()
+ g:yql
+)
+
+SRCS(
+ dq_solomon_shard.proto
+)
+
+EXCLUDE_TAGS(GO_PROTO)
+
+END()
diff --git a/ydb/library/yql/providers/solomon/provider/ya.make b/ydb/library/yql/providers/solomon/provider/ya.make
index c81aa05a3df..b80c01bdb74 100644
--- a/ydb/library/yql/providers/solomon/provider/ya.make
+++ b/ydb/library/yql/providers/solomon/provider/ya.make
@@ -7,21 +7,21 @@ OWNER(
SRCS(
yql_solomon_config.cpp
- yql_solomon_datasink_execution.cpp
- yql_solomon_datasink_type_ann.cpp
- yql_solomon_datasink.cpp
- yql_solomon_datasource_execution.cpp
- yql_solomon_datasource_type_ann.cpp
- yql_solomon_datasource.cpp
- yql_solomon_dq_integration.cpp
+ yql_solomon_datasink_execution.cpp
+ yql_solomon_datasink_type_ann.cpp
+ yql_solomon_datasink.cpp
+ yql_solomon_datasource_execution.cpp
+ yql_solomon_datasource_type_ann.cpp
+ yql_solomon_datasource.cpp
+ yql_solomon_dq_integration.cpp
yql_solomon_io_discovery.cpp
yql_solomon_load_meta.cpp
- yql_solomon_physical_optimize.cpp
- yql_solomon_provider.cpp
+ yql_solomon_physical_optimize.cpp
+ yql_solomon_provider.cpp
)
PEERDIR(
- library/cpp/actors/protos
+ library/cpp/actors/protos
ydb/library/yql/dq/expr_nodes
ydb/library/yql/providers/common/config
ydb/library/yql/providers/common/proto
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h
index 2c24d3c2a20..941b2adc305 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_config.h
@@ -21,13 +21,13 @@ struct TSolomonConfiguration
TSolomonConfiguration(const TSolomonConfiguration&) = delete;
template <typename TProtoConfig>
- void Init(const TProtoConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx)
+ void Init(const TProtoConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx)
{
TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
for (auto& cluster: config.GetClusterMapping()) {
clusters.push_back(cluster.GetName());
- ClusterConfigs[cluster.GetName()] = cluster;
-
+ ClusterConfigs[cluster.GetName()] = cluster;
+
const TString authToken = typeCtx->FindCredentialContent("cluster:default_" + cluster.GetName(), "default_solomon", cluster.GetToken());
Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken);
}
@@ -42,9 +42,9 @@ struct TSolomonConfiguration
}
TSolomonSettings::TConstPtr Snapshot() const;
-
- THashMap<TString, TSolomonClusterConfig> ClusterConfigs;
- THashMap<TString, TString> Tokens;
+
+ THashMap<TString, TSolomonClusterConfig> ClusterConfigs;
+ THashMap<TString, TString> Tokens;
};
} // NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
index 2c8d18ee82f..b09fd381a07 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
@@ -19,7 +19,7 @@ public:
: State_ {state}
, TypeAnnotationTransformer_(CreateSolomonDataSinkTypeAnnotationTransformer(State_))
, ExecutionTransformer_(CreateSolomonDataSinkExecTransformer(State_))
- , PhysicalOptProposalTransformer_(CreateSoPhysicalOptProposalTransformer(State_))
+ , PhysicalOptProposalTransformer_(CreateSoPhysicalOptProposalTransformer(State_))
{
}
@@ -39,10 +39,10 @@ public:
.Done().Ptr();
}
- const THashMap<TString, TString>* GetClusterTokens() override {
- return &State_->Configuration->Tokens;
- }
-
+ const THashMap<TString, TString>* GetClusterTokens() override {
+ return &State_->Configuration->Tokens;
+ }
+
IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
Y_UNUSED(instantOnly);
return *TypeAnnotationTransformer_;
@@ -52,10 +52,10 @@ public:
return *ExecutionTransformer_;
}
- IGraphTransformer& GetPhysicalOptProposalTransformer() override {
- return *PhysicalOptProposalTransformer_;
- }
-
+ IGraphTransformer& GetPhysicalOptProposalTransformer() override {
+ return *PhysicalOptProposalTransformer_;
+ }
+
bool CanParse(const TExprNode& node) override {
if (node.IsCallable(TCoWrite::CallableName())) {
return TSoDataSink::Match(node.Child(1));
@@ -160,16 +160,16 @@ public:
return TStringBuilder() << SolomonProviderName << '.' << node.Child(1)->Content();
}
- IDqIntegration* GetDqIntegration() override {
- return State_->IsRtmrMode() ? nullptr : State_->DqIntegration.Get();
- }
-
+ IDqIntegration* GetDqIntegration() override {
+ return State_->IsRtmrMode() ? nullptr : State_->DqIntegration.Get();
+ }
+
private:
TSolomonState::TPtr State_;
THolder<TVisitorTransformerBase> TypeAnnotationTransformer_;
THolder<TExecTransformerBase> ExecutionTransformer_;
- THolder<IGraphTransformer> PhysicalOptProposalTransformer_;
+ THolder<IGraphTransformer> PhysicalOptProposalTransformer_;
};
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp
index 8b4c7945295..e3048498c7b 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_execution.cpp
@@ -8,126 +8,126 @@
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h>
#include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
-#include <util/string/split.h>
-
+#include <util/string/split.h>
+
namespace NYql {
using namespace NNodes;
class TSolomonDataSinkExecTransformer : public TExecTransformerBase {
public:
- explicit TSolomonDataSinkExecTransformer(TSolomonState::TPtr state)
+ explicit TSolomonDataSinkExecTransformer(TSolomonState::TPtr state)
: State_(state)
{
- AddHandler({TCoCommit::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleCommit));
- AddHandler({TSoWriteToShard::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleSoWriteToShard));
+ AddHandler({TCoCommit::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleCommit));
+ AddHandler({TSoWriteToShard::CallableName()}, RequireFirst(), Hndl(&TSolomonDataSinkExecTransformer::HandleSoWriteToShard));
+ }
+
+ TStatusCallbackPair HandleCommit(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!State_->IsRtmrMode() && TDqQuery::Match(input->Child(TCoCommit::idx_World))) {
+ return DelegateExecutionToDqProvider(input->ChildPtr(TCoCommit::idx_World), input, ctx);
+ }
+
+ return ExecPass(input, ctx);
+ }
+
+ TStatusCallbackPair HandleSoWriteToShard(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (State_->IsRtmrMode()) {
+ return ExecPass(input, ctx);
+ }
+
+ return DelegateExecutionToDqProvider(input->ChildPtr(TSoWriteToShard::idx_Input), input, ctx);
}
- TStatusCallbackPair HandleCommit(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!State_->IsRtmrMode() && TDqQuery::Match(input->Child(TCoCommit::idx_World))) {
- return DelegateExecutionToDqProvider(input->ChildPtr(TCoCommit::idx_World), input, ctx);
- }
-
- return ExecPass(input, ctx);
- }
-
- TStatusCallbackPair HandleSoWriteToShard(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (State_->IsRtmrMode()) {
- return ExecPass(input, ctx);
- }
-
- return DelegateExecutionToDqProvider(input->ChildPtr(TSoWriteToShard::idx_Input), input, ctx);
- }
-
private:
- TStatusCallbackPair DelegateExecutionToDqProvider(
- const TExprNode::TPtr& input,
- const TExprNode::TPtr& originInput,
- TExprContext& ctx)
- {
- YQL_CLOG(INFO, ProviderSolomon) << "Delegate execution of " << input->Content() << " to dq provider";
- auto delegatedNode = Build<TPull>(ctx, input->Pos())
- .Input(input)
- .BytesLimit()
- .Value(TString())
- .Build()
- .RowsLimit()
- .Value(TString("0"))
- .Build()
- .FormatDetails()
+ TStatusCallbackPair DelegateExecutionToDqProvider(
+ const TExprNode::TPtr& input,
+ const TExprNode::TPtr& originInput,
+ TExprContext& ctx)
+ {
+ YQL_CLOG(INFO, ProviderSolomon) << "Delegate execution of " << input->Content() << " to dq provider";
+ auto delegatedNode = Build<TPull>(ctx, input->Pos())
+ .Input(input)
+ .BytesLimit()
+ .Value(TString())
+ .Build()
+ .RowsLimit()
+ .Value(TString("0"))
+ .Build()
+ .FormatDetails()
.Value(ToString((ui32)NYson::EYsonFormat::Binary))
- .Build()
- .Settings()
- .Build()
- .Format()
- .Value(ToString("0"))
- .Build()
- .PublicId()
- .Value("id")
- .Build()
- .Discard()
- .Value(ToString(true))
- .Build()
- .Origin(originInput)
- .Done()
- .Ptr();
-
- auto atomType = ctx.MakeType<TUnitExprType>();
-
- for (auto idx: {TResOrPullBase::idx_BytesLimit, TResOrPullBase::idx_RowsLimit, TResOrPullBase::idx_FormatDetails,
- TResOrPullBase::idx_Format, TResOrPullBase::idx_PublicId, TResOrPullBase::idx_Discard }) {
- delegatedNode->Child(idx)->SetTypeAnn(atomType);
- delegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete);
- }
-
- delegatedNode->SetTypeAnn(originInput->GetTypeAnn());
- delegatedNode->SetState(TExprNode::EState::ConstrComplete);
- originInput->SetState(TExprNode::EState::ExecutionInProgress);
-
- auto dqProvider = State_->Types->DataSourceMap.FindPtr(DqProviderName);
- YQL_ENSURE(dqProvider);
-
- TExprNode::TPtr delegatedNodeOutput;
- auto status = dqProvider->Get()->GetCallableExecutionTransformer().Transform(delegatedNode, delegatedNodeOutput, ctx);
-
- if (status.Level != TStatus::Async) {
- YQL_ENSURE(status.Level != TStatus::Ok, "Asynchronous execution is expected in a happy path.");
- return SyncStatus(status);
- }
-
- auto dqFuture = dqProvider->Get()->GetCallableExecutionTransformer().GetAsyncFuture(*delegatedNode);
-
- TAsyncTransformCallbackFuture callbackFuture = dqFuture.Apply(
- [dqProvider, delegatedNode](const NThreading::TFuture<void>& completedFuture) {
- return TAsyncTransformCallback(
- [completedFuture, dqProvider, delegatedNode](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- completedFuture.GetValue();
- TExprNode::TPtr delegatedNodeOutput;
- auto dqWriteStatus = dqProvider->Get()->GetCallableExecutionTransformer()
- .ApplyAsyncChanges(delegatedNode, delegatedNodeOutput, ctx);
-
- YQL_ENSURE(dqWriteStatus != TStatus::Async, "ApplyAsyncChanges should not return Async.");
-
- if (dqWriteStatus != TStatus::Ok) {
- output = input;
- return dqWriteStatus;
- }
-
- input->SetState(TExprNode::EState::ExecutionComplete);
- output = ctx.ShallowCopy(*input);
- output->SetResult(ctx.NewAtom(input->Pos(), "DQ_completed"));
-
- return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true);
- });
- });
-
- return std::make_pair(IGraphTransformer::TStatus::Async, callbackFuture);
- }
-
-private:
+ .Build()
+ .Settings()
+ .Build()
+ .Format()
+ .Value(ToString("0"))
+ .Build()
+ .PublicId()
+ .Value("id")
+ .Build()
+ .Discard()
+ .Value(ToString(true))
+ .Build()
+ .Origin(originInput)
+ .Done()
+ .Ptr();
+
+ auto atomType = ctx.MakeType<TUnitExprType>();
+
+ for (auto idx: {TResOrPullBase::idx_BytesLimit, TResOrPullBase::idx_RowsLimit, TResOrPullBase::idx_FormatDetails,
+ TResOrPullBase::idx_Format, TResOrPullBase::idx_PublicId, TResOrPullBase::idx_Discard }) {
+ delegatedNode->Child(idx)->SetTypeAnn(atomType);
+ delegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete);
+ }
+
+ delegatedNode->SetTypeAnn(originInput->GetTypeAnn());
+ delegatedNode->SetState(TExprNode::EState::ConstrComplete);
+ originInput->SetState(TExprNode::EState::ExecutionInProgress);
+
+ auto dqProvider = State_->Types->DataSourceMap.FindPtr(DqProviderName);
+ YQL_ENSURE(dqProvider);
+
+ TExprNode::TPtr delegatedNodeOutput;
+ auto status = dqProvider->Get()->GetCallableExecutionTransformer().Transform(delegatedNode, delegatedNodeOutput, ctx);
+
+ if (status.Level != TStatus::Async) {
+ YQL_ENSURE(status.Level != TStatus::Ok, "Asynchronous execution is expected in a happy path.");
+ return SyncStatus(status);
+ }
+
+ auto dqFuture = dqProvider->Get()->GetCallableExecutionTransformer().GetAsyncFuture(*delegatedNode);
+
+ TAsyncTransformCallbackFuture callbackFuture = dqFuture.Apply(
+ [dqProvider, delegatedNode](const NThreading::TFuture<void>& completedFuture) {
+ return TAsyncTransformCallback(
+ [completedFuture, dqProvider, delegatedNode](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ completedFuture.GetValue();
+ TExprNode::TPtr delegatedNodeOutput;
+ auto dqWriteStatus = dqProvider->Get()->GetCallableExecutionTransformer()
+ .ApplyAsyncChanges(delegatedNode, delegatedNodeOutput, ctx);
+
+ YQL_ENSURE(dqWriteStatus != TStatus::Async, "ApplyAsyncChanges should not return Async.");
+
+ if (dqWriteStatus != TStatus::Ok) {
+ output = input;
+ return dqWriteStatus;
+ }
+
+ input->SetState(TExprNode::EState::ExecutionComplete);
+ output = ctx.ShallowCopy(*input);
+ output->SetResult(ctx.NewAtom(input->Pos(), "DQ_completed"));
+
+ return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true);
+ });
+ });
+
+ return std::make_pair(IGraphTransformer::TStatus::Async, callbackFuture);
+ }
+
+private:
TSolomonState::TPtr State_;
};
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp
index b0043b9cd32..dc9ea45a8f6 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink_type_ann.cpp
@@ -14,7 +14,7 @@ public:
{
using TSelf = TSolomonDataSinkTypeAnnotationTransformer;
AddHandler({TSoWriteToShard::CallableName()}, Hndl(&TSelf::HandleWriteToShard));
- AddHandler({TSoShard::CallableName()}, Hndl(&TSelf::HandleSoShard));
+ AddHandler({TSoShard::CallableName()}, Hndl(&TSelf::HandleSoShard));
AddHandler({TCoCommit::CallableName()}, Hndl(&TSelf::HandleCommit));
}
@@ -37,39 +37,39 @@ private:
return TStatus::Ok;
}
- TStatus HandleSoShard(TExprBase input, TExprContext& ctx) {
- YQL_ENSURE(!State_->IsRtmrMode(), "SoShard can't be used in rtmr mode");
-
- if (!EnsureMinArgsCount(input.Ref(), 4, ctx) || !EnsureMaxArgsCount(input.Ref(), 5, ctx)) {
- return TStatus::Error;
- }
-
- const TSoShard shard = input.Cast<TSoShard>();
-
- if (!EnsureAtom(shard.SolomonCluster().Ref(), ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureAtom(shard.Project().Ref(), ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureAtom(shard.Cluster().Ref(), ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureAtom(shard.Service().Ref(), ctx)) {
- return TStatus::Error;
- }
-
- if (shard.Token() && !EnsureCallable(shard.Token().Ref(), ctx)) {
- return TStatus::Error;
- }
-
- input.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>());
- return TStatus::Ok;
- }
-
+ TStatus HandleSoShard(TExprBase input, TExprContext& ctx) {
+ YQL_ENSURE(!State_->IsRtmrMode(), "SoShard can't be used in rtmr mode");
+
+ if (!EnsureMinArgsCount(input.Ref(), 4, ctx) || !EnsureMaxArgsCount(input.Ref(), 5, ctx)) {
+ return TStatus::Error;
+ }
+
+ const TSoShard shard = input.Cast<TSoShard>();
+
+ if (!EnsureAtom(shard.SolomonCluster().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureAtom(shard.Project().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureAtom(shard.Cluster().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureAtom(shard.Service().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (shard.Token() && !EnsureCallable(shard.Token().Ref(), ctx)) {
+ return TStatus::Error;
+ }
+
+ input.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>());
+ return TStatus::Ok;
+ }
+
TStatus HandleCommit(TExprBase input, TExprContext& ctx) {
Y_UNUSED(ctx);
auto commit = input.Cast<TCoCommit>();
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp
index c93cbd3cd2d..773c9609914 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp
@@ -1,5 +1,5 @@
-#include "yql_solomon_dq_integration.h"
-
+#include "yql_solomon_dq_integration.h"
+
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
#include <ydb/library/yql/utils/log/log.h>
@@ -9,150 +9,150 @@
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h>
#include <ydb/library/yql/providers/solomon/proto/dq_solomon_shard.pb.h>
-
-#include <util/string/builder.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
-namespace {
-
-constexpr i32 MaxLabelsCount = 16;
-constexpr i32 MaxSensorsCount = 50;
-
-NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) {
- switch (clusterType) {
- case TSolomonClusterConfig::SCT_SOLOMON:
- return NSo::NProto::ESolomonClusterType::CT_SOLOMON;
- case TSolomonClusterConfig::SCT_MONITORING:
- return NSo::NProto::ESolomonClusterType::CT_MONITORING;
- default:
- YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterType));
- }
-}
-
+
+#include <util/string/builder.h>
+
+namespace NYql {
+
+using namespace NNodes;
+
+namespace {
+
+constexpr i32 MaxLabelsCount = 16;
+constexpr i32 MaxSensorsCount = 50;
+
+NSo::NProto::ESolomonClusterType MapClusterType(TSolomonClusterConfig::ESolomonClusterType clusterType) {
+ switch (clusterType) {
+ case TSolomonClusterConfig::SCT_SOLOMON:
+ return NSo::NProto::ESolomonClusterType::CT_SOLOMON;
+ case TSolomonClusterConfig::SCT_MONITORING:
+ return NSo::NProto::ESolomonClusterType::CT_MONITORING;
+ default:
+ YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterType));
+ }
+}
+
const TTypeAnnotationNode* GetItemType(const TExprNode& node) {
- const TTypeAnnotationNode* typeAnn = node.GetTypeAnn();
- switch (typeAnn->GetKind()) {
- case ETypeAnnotationKind::Flow:
+ const TTypeAnnotationNode* typeAnn = node.GetTypeAnn();
+ switch (typeAnn->GetKind()) {
+ case ETypeAnnotationKind::Flow:
return typeAnn->Cast<TFlowExprType>()->GetItemType();
- case ETypeAnnotationKind::Stream:
+ case ETypeAnnotationKind::Stream:
return typeAnn->Cast<TStreamExprType>()->GetItemType();
- default: break;
- }
- YQL_ENSURE(false, "Invalid solomon sink type " << typeAnn->GetKind());
+ default: break;
+ }
+ YQL_ENSURE(false, "Invalid solomon sink type " << typeAnn->GetKind());
return nullptr;
-}
-
-void FillScheme(const TTypeAnnotationNode& itemType, NSo::NProto::TDqSolomonShardScheme& scheme) {
- int index = 0;
- for (const TItemExprType* structItem : itemType.Cast<TStructExprType>()->GetItems()) {
- const auto itemName = structItem->GetName();
- const auto* itemType = structItem->GetItemType();
- YQL_ENSURE(
- itemType->GetKind() != ETypeAnnotationKind::Optional,
- "Optional types are not supported in monitoring sink. FieldName: " << itemName);
-
- const auto dataType = NUdf::GetDataTypeInfo(itemType->Cast<TDataExprType>()->GetSlot());
-
- NSo::NProto::TDqSolomonSchemeItem schemeItem;
- schemeItem.SetKey(TString(itemName));
- schemeItem.SetIndex(index++);
- schemeItem.SetDataTypeId(dataType.TypeId);
-
- if (dataType.Features & NUdf::DateType || dataType.Features & NUdf::TzDateType) {
- YQL_ENSURE(!scheme.HasTimestamp(), "Multiple timestamps were provided for monitoing sink");
- *scheme.MutableTimestamp() = schemeItem;
- } else if (dataType.Features & NUdf::NumericType) {
- scheme.MutableSensors()->Add(std::move(schemeItem));
- } else if (dataType.Features & NUdf::StringType) {
- scheme.MutableLabels()->Add(std::move(schemeItem));
- } else {
- YQL_ENSURE(false, "Ivalid data type for monitoing sink: " << dataType.Name);
- }
- }
-
- YQL_ENSURE(scheme.HasTimestamp(), "Timestamp wasn't provided for monitoing sink");
- YQL_ENSURE(!scheme.GetSensors().empty(), "No sensors were provided for monitoing sink");
- YQL_ENSURE(!scheme.GetLabels().empty(), "No labels were provided for monitoing sink");
-
- YQL_ENSURE(scheme.GetLabels().size() <= MaxLabelsCount,
- "Max labels count is " << MaxLabelsCount << " but " << scheme.GetLabels().size() << " were provided");
- YQL_ENSURE(scheme.GetSensors().size() <= MaxSensorsCount,
- "Max sensors count is " << MaxSensorsCount << " but " << scheme.GetSensors().size() << " were provided");
-}
-
-class TSolomonDqIntegration: public TDqIntegrationBase {
-public:
- TSolomonDqIntegration(const TSolomonState::TPtr& state)
- : State_(state.Get())
- {
- }
-
- TMaybe<ui64> CanRead(const TDqSettings&, const TExprNode&, TExprContext&, bool) override {
- YQL_ENSURE(false, "Unimplemented");
- }
-
- TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr&, TExprContext&) override {
- YQL_ENSURE(false, "Unimplemented");
- }
-
+}
+
+void FillScheme(const TTypeAnnotationNode& itemType, NSo::NProto::TDqSolomonShardScheme& scheme) {
+ int index = 0;
+ for (const TItemExprType* structItem : itemType.Cast<TStructExprType>()->GetItems()) {
+ const auto itemName = structItem->GetName();
+ const auto* itemType = structItem->GetItemType();
+ YQL_ENSURE(
+ itemType->GetKind() != ETypeAnnotationKind::Optional,
+ "Optional types are not supported in monitoring sink. FieldName: " << itemName);
+
+ const auto dataType = NUdf::GetDataTypeInfo(itemType->Cast<TDataExprType>()->GetSlot());
+
+ NSo::NProto::TDqSolomonSchemeItem schemeItem;
+ schemeItem.SetKey(TString(itemName));
+ schemeItem.SetIndex(index++);
+ schemeItem.SetDataTypeId(dataType.TypeId);
+
+ if (dataType.Features & NUdf::DateType || dataType.Features & NUdf::TzDateType) {
+ YQL_ENSURE(!scheme.HasTimestamp(), "Multiple timestamps were provided for monitoing sink");
+ *scheme.MutableTimestamp() = schemeItem;
+ } else if (dataType.Features & NUdf::NumericType) {
+ scheme.MutableSensors()->Add(std::move(schemeItem));
+ } else if (dataType.Features & NUdf::StringType) {
+ scheme.MutableLabels()->Add(std::move(schemeItem));
+ } else {
+ YQL_ENSURE(false, "Ivalid data type for monitoing sink: " << dataType.Name);
+ }
+ }
+
+ YQL_ENSURE(scheme.HasTimestamp(), "Timestamp wasn't provided for monitoing sink");
+ YQL_ENSURE(!scheme.GetSensors().empty(), "No sensors were provided for monitoing sink");
+ YQL_ENSURE(!scheme.GetLabels().empty(), "No labels were provided for monitoing sink");
+
+ YQL_ENSURE(scheme.GetLabels().size() <= MaxLabelsCount,
+ "Max labels count is " << MaxLabelsCount << " but " << scheme.GetLabels().size() << " were provided");
+ YQL_ENSURE(scheme.GetSensors().size() <= MaxSensorsCount,
+ "Max sensors count is " << MaxSensorsCount << " but " << scheme.GetSensors().size() << " were provided");
+}
+
+class TSolomonDqIntegration: public TDqIntegrationBase {
+public:
+ TSolomonDqIntegration(const TSolomonState::TPtr& state)
+ : State_(state.Get())
+ {
+ }
+
+ TMaybe<ui64> CanRead(const TDqSettings&, const TExprNode&, TExprContext&, bool) override {
+ YQL_ENSURE(false, "Unimplemented");
+ }
+
+ TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr&, TExprContext&) override {
+ YQL_ENSURE(false, "Unimplemented");
+ }
+
TMaybe<bool> CanWrite(const TDqSettings&, const TExprNode&, TExprContext&) override {
- YQL_ENSURE(false, "Unimplemented");
- }
-
- void FillSourceSettings(const TExprNode&, ::google::protobuf::Any&, TString& ) override {
- YQL_ENSURE(false, "Unimplemented");
- }
-
- void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& protoSettings, TString& sinkType) override {
- const auto maybeDqSink = TMaybeNode<TDqSink>(&node);
- if (!maybeDqSink) {
- return;
- }
- const auto dqSink = maybeDqSink.Cast();
-
- const auto settings = dqSink.Settings();
- const auto maybeShard = TMaybeNode<TSoShard>(settings.Raw());
- if (!maybeShard) {
- return;
- }
-
- const TSoShard shard = maybeShard.Cast();
-
- const auto solomonCluster = shard.SolomonCluster().StringValue();
- const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(solomonCluster);
- YQL_ENSURE(clusterDesc, "Unknown cluster " << solomonCluster);
-
- NSo::NProto::TDqSolomonShard shardDesc;
- shardDesc.SetEndpoint(clusterDesc->GetCluster());
- shardDesc.SetProject(shard.Project().StringValue());
- shardDesc.SetCluster(shard.Cluster().StringValue());
- shardDesc.SetService(shard.Service().StringValue());
-
- shardDesc.SetClusterType(MapClusterType(clusterDesc->GetClusterType()));
+ YQL_ENSURE(false, "Unimplemented");
+ }
+
+ void FillSourceSettings(const TExprNode&, ::google::protobuf::Any&, TString& ) override {
+ YQL_ENSURE(false, "Unimplemented");
+ }
+
+ void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& protoSettings, TString& sinkType) override {
+ const auto maybeDqSink = TMaybeNode<TDqSink>(&node);
+ if (!maybeDqSink) {
+ return;
+ }
+ const auto dqSink = maybeDqSink.Cast();
+
+ const auto settings = dqSink.Settings();
+ const auto maybeShard = TMaybeNode<TSoShard>(settings.Raw());
+ if (!maybeShard) {
+ return;
+ }
+
+ const TSoShard shard = maybeShard.Cast();
+
+ const auto solomonCluster = shard.SolomonCluster().StringValue();
+ const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(solomonCluster);
+ YQL_ENSURE(clusterDesc, "Unknown cluster " << solomonCluster);
+
+ NSo::NProto::TDqSolomonShard shardDesc;
+ shardDesc.SetEndpoint(clusterDesc->GetCluster());
+ shardDesc.SetProject(shard.Project().StringValue());
+ shardDesc.SetCluster(shard.Cluster().StringValue());
+ shardDesc.SetService(shard.Service().StringValue());
+
+ shardDesc.SetClusterType(MapClusterType(clusterDesc->GetClusterType()));
shardDesc.SetUseSsl(clusterDesc->GetUseSsl());
-
+
const TTypeAnnotationNode* itemType = GetItemType(node);
- FillScheme(*itemType, *shardDesc.MutableScheme());
-
+ FillScheme(*itemType, *shardDesc.MutableScheme());
+
if (auto maybeToken = shard.Token()) {
shardDesc.MutableToken()->SetName(TString(maybeToken.Cast().Name().Value()));
- }
-
- protoSettings.PackFrom(shardDesc);
- sinkType = "SolomonSink";
- }
-
-private:
- TSolomonState* State_; // State owns dq integration, so back reference must be not smart.
-};
-
-}
-
-THolder<IDqIntegration> CreateSolomonDqIntegration(const TSolomonState::TPtr& state) {
+ }
+
+ protoSettings.PackFrom(shardDesc);
+ sinkType = "SolomonSink";
+ }
+
+private:
+ TSolomonState* State_; // State owns dq integration, so back reference must be not smart.
+};
+
+}
+
+THolder<IDqIntegration> CreateSolomonDqIntegration(const TSolomonState::TPtr& state) {
return MakeHolder<TSolomonDqIntegration>(state);
-}
-
-}
+}
+
+}
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h
index 3355341af97..7a621caff16 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.h
@@ -1,13 +1,13 @@
-#pragma once
-
-#include "yql_solomon_provider.h"
-
+#pragma once
+
+#include "yql_solomon_provider.h"
+
#include <ydb/library/yql/providers/dq/interface/yql_dq_integration.h>
-
-#include <util/generic/ptr.h>
-
-namespace NYql {
-
-THolder<IDqIntegration> CreateSolomonDqIntegration(const TSolomonState::TPtr& state);
-
-}
+
+#include <util/generic/ptr.h>
+
+namespace NYql {
+
+THolder<IDqIntegration> CreateSolomonDqIntegration(const TSolomonState::TPtr& state);
+
+}
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp
index ad60fc7b378..226205cce20 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_physical_optimize.cpp
@@ -1,5 +1,5 @@
-#include "yql_solomon_provider_impl.h"
-
+#include "yql_solomon_provider_impl.h"
+
#include <ydb/library/yql/core/yql_opt_utils.h>
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
#include <ydb/library/yql/dq/opt/dq_opt.h>
@@ -9,168 +9,168 @@
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.h>
#include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h>
-
-#include <util/string/split.h>
-
-namespace NYql {
-
-namespace {
-
-using namespace NNodes;
-using namespace NDq;
-
-TString FormatShardPathInvalid(const TSolomonClusterConfig& clusterConfig, const TString& path) {
- TStringBuilder err;
- err << "Invalid shard path " << path << " It should be ";
- switch (clusterConfig.GetClusterType()) {
- case TSolomonClusterConfig::SCT_SOLOMON:
- err << "{project}/{cluster}/{service}";
- break;
- case TSolomonClusterConfig::SCT_MONITORING:
- err << "{cloudId}/{folderId}/{service}";
- break;
- default:
- YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterConfig.GetClusterType()));
- }
- err << " or just {service} when using connection.";
- return err;
-}
-
-void ParseShardPath(
- const TSolomonClusterConfig& clusterConfig,
- const TString& path,
- TString& project,
- TString& cluster,
- TString& service)
-{
- std::vector<TString> shardData;
- shardData.reserve(3);
- for (const auto& it : StringSplitter(path).Split('/')) {
- shardData.emplace_back(it.Token());
- }
- YQL_ENSURE(shardData.size() == 1 || shardData.size() == 3, "" << FormatShardPathInvalid(clusterConfig, path));
-
- project = shardData.size() == 3 ? shardData.at(0) : TString();
- cluster = shardData.size() == 3 ? shardData.at(1) : TString();
- service = shardData.back();
-}
-
-class TSoPhysicalOptProposalTransformer : public TOptimizeTransformerBase {
-public:
- explicit TSoPhysicalOptProposalTransformer(TSolomonState::TPtr state)
- : TOptimizeTransformerBase(state->Types, NLog::EComponent::ProviderSolomon, {})
- , State_(std::move(state))
- {
-#define HNDL(name) "PhysicalOptimizer-"#name, Hndl(&TSoPhysicalOptProposalTransformer::name)
- AddHandler(0, &TSoWriteToShard::Match, HNDL(SoWriteToShard));
-#undef HNDL
+
+#include <util/string/split.h>
+
+namespace NYql {
+
+namespace {
+
+using namespace NNodes;
+using namespace NDq;
+
+TString FormatShardPathInvalid(const TSolomonClusterConfig& clusterConfig, const TString& path) {
+ TStringBuilder err;
+ err << "Invalid shard path " << path << " It should be ";
+ switch (clusterConfig.GetClusterType()) {
+ case TSolomonClusterConfig::SCT_SOLOMON:
+ err << "{project}/{cluster}/{service}";
+ break;
+ case TSolomonClusterConfig::SCT_MONITORING:
+ err << "{cloudId}/{folderId}/{service}";
+ break;
+ default:
+ YQL_ENSURE(false, "Invalid cluster type " << ToString<ui32>(clusterConfig.GetClusterType()));
+ }
+ err << " or just {service} when using connection.";
+ return err;
+}
+
+void ParseShardPath(
+ const TSolomonClusterConfig& clusterConfig,
+ const TString& path,
+ TString& project,
+ TString& cluster,
+ TString& service)
+{
+ std::vector<TString> shardData;
+ shardData.reserve(3);
+ for (const auto& it : StringSplitter(path).Split('/')) {
+ shardData.emplace_back(it.Token());
+ }
+ YQL_ENSURE(shardData.size() == 1 || shardData.size() == 3, "" << FormatShardPathInvalid(clusterConfig, path));
+
+ project = shardData.size() == 3 ? shardData.at(0) : TString();
+ cluster = shardData.size() == 3 ? shardData.at(1) : TString();
+ service = shardData.back();
+}
+
+class TSoPhysicalOptProposalTransformer : public TOptimizeTransformerBase {
+public:
+ explicit TSoPhysicalOptProposalTransformer(TSolomonState::TPtr state)
+ : TOptimizeTransformerBase(state->Types, NLog::EComponent::ProviderSolomon, {})
+ , State_(std::move(state))
+ {
+#define HNDL(name) "PhysicalOptimizer-"#name, Hndl(&TSoPhysicalOptProposalTransformer::name)
+ AddHandler(0, &TSoWriteToShard::Match, HNDL(SoWriteToShard));
+#undef HNDL
SetGlobal(0); // Stage 0 of this optimizer is global => we can remap nodes.
- }
-
- TMaybeNode<TExprBase> SoWriteToShard(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) const {
- if (State_->IsRtmrMode()) {
- return node;
- }
-
- auto write = node.Cast<TSoWriteToShard>();
- if (!TDqCnUnionAll::Match(write.Input().Raw())) {
- // If input is not DqCnUnionAll, it means not all dq optimizations are done yet
- return node;
- }
-
- const TParentsMap* parentsMap = getParents();
- auto dqUnion = write.Input().Cast<TDqCnUnionAll>();
- if (!NDq::IsSingleConsumerConnection(dqUnion, *parentsMap)) {
- return node;
- }
-
- YQL_CLOG(INFO, ProviderSolomon) << "Optimize SoWriteToShard";
-
- const auto solomonCluster = TString(write.DataSink().Cluster().Value());
- auto shard = BuildSolomonShard(write.Shard().Cast<TCoAtom>(), ctx, solomonCluster);
-
- auto dqSink = Build<TDqSink>(ctx, write.Pos())
- .DataSink(write.DataSink())
- .Settings(shard)
- .Index(dqUnion.Output().Index())
- .Done();
-
- TDqStage inputStage = dqUnion.Output().Stage().Cast<TDqStage>();
-
+ }
+
+ TMaybeNode<TExprBase> SoWriteToShard(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) const {
+ if (State_->IsRtmrMode()) {
+ return node;
+ }
+
+ auto write = node.Cast<TSoWriteToShard>();
+ if (!TDqCnUnionAll::Match(write.Input().Raw())) {
+ // If input is not DqCnUnionAll, it means not all dq optimizations are done yet
+ return node;
+ }
+
+ const TParentsMap* parentsMap = getParents();
+ auto dqUnion = write.Input().Cast<TDqCnUnionAll>();
+ if (!NDq::IsSingleConsumerConnection(dqUnion, *parentsMap)) {
+ return node;
+ }
+
+ YQL_CLOG(INFO, ProviderSolomon) << "Optimize SoWriteToShard";
+
+ const auto solomonCluster = TString(write.DataSink().Cluster().Value());
+ auto shard = BuildSolomonShard(write.Shard().Cast<TCoAtom>(), ctx, solomonCluster);
+
+ auto dqSink = Build<TDqSink>(ctx, write.Pos())
+ .DataSink(write.DataSink())
+ .Settings(shard)
+ .Index(dqUnion.Output().Index())
+ .Done();
+
+ TDqStage inputStage = dqUnion.Output().Stage().Cast<TDqStage>();
+
auto sinksBuilder = Build<TDqSinksList>(ctx, inputStage.Pos());
if (inputStage.Sinks()) {
sinksBuilder.InitFrom(inputStage.Sinks().Cast());
}
sinksBuilder.Add(dqSink);
-
+
auto dqStageWithSink = Build<TDqStage>(ctx, inputStage.Pos())
.InitFrom(inputStage)
.Sinks(sinksBuilder.Done())
.Done();
-
- auto dqQueryBuilder = Build<TDqQuery>(ctx, write.Pos());
- dqQueryBuilder.World(write.World());
- dqQueryBuilder.SinkStages().Add(dqStageWithSink).Build();
+
+ auto dqQueryBuilder = Build<TDqQuery>(ctx, write.Pos());
+ dqQueryBuilder.World(write.World());
+ dqQueryBuilder.SinkStages().Add(dqStageWithSink).Build();
optCtx.RemapNode(inputStage.Ref(), dqStageWithSink.Ptr());
- return dqQueryBuilder.Done();
- }
-
-private:
- TExprBase BuildSolomonShard(TCoAtom shardNode, TExprContext& ctx, TString solomonCluster) const {
- const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(solomonCluster);
- YQL_ENSURE(clusterDesc, "Unknown cluster " << solomonCluster);
-
- TString project, cluster, service;
- ParseShardPath(*clusterDesc, shardNode.StringValue(), project, cluster, service);
-
- if (project.empty() && clusterDesc->HasPath()) {
- project = clusterDesc->GetPath().GetProject();
- }
- YQL_ENSURE(!project.empty(), "Project is not defined. You can define it inside connection, or inside query.");
-
- if (cluster.empty() && clusterDesc->HasPath()) {
- cluster = clusterDesc->GetPath().GetCluster();
- }
- YQL_ENSURE(!cluster.empty(), "Cluster is not defined. You can define it inside connection, or inside query.");
-
- auto solomonClusterAtom = Build<TCoAtom>(ctx, shardNode.Pos())
- .Value(solomonCluster)
- .Done();
-
- auto projectAtom = Build<TCoAtom>(ctx, shardNode.Pos())
- .Value(project)
- .Done();
-
- auto clusterAtom = Build<TCoAtom>(ctx, shardNode.Pos())
- .Value(cluster)
- .Done();
-
- auto serviceAtom = Build<TCoAtom>(ctx, shardNode.Pos())
- .Value(service)
- .Done();
-
- auto dqSoShardBuilder = Build<TSoShard>(ctx, shardNode.Pos());
- dqSoShardBuilder.SolomonCluster(solomonClusterAtom);
- dqSoShardBuilder.Project(projectAtom);
- dqSoShardBuilder.Cluster(clusterAtom);
- dqSoShardBuilder.Service(serviceAtom);
-
- dqSoShardBuilder.Token<TCoSecureParam>().Name().Build("cluster:default_" + solomonCluster).Build();
-
- return dqSoShardBuilder.Done();
- }
-
-private:
- TSolomonState::TPtr State_;
-};
-
-} // namespace
-
-THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state) {
- return MakeHolder<TSoPhysicalOptProposalTransformer>(std::move(state));
-}
-
-} // namespace NYql
+ return dqQueryBuilder.Done();
+ }
+
+private:
+ TExprBase BuildSolomonShard(TCoAtom shardNode, TExprContext& ctx, TString solomonCluster) const {
+ const auto* clusterDesc = State_->Configuration->ClusterConfigs.FindPtr(solomonCluster);
+ YQL_ENSURE(clusterDesc, "Unknown cluster " << solomonCluster);
+
+ TString project, cluster, service;
+ ParseShardPath(*clusterDesc, shardNode.StringValue(), project, cluster, service);
+
+ if (project.empty() && clusterDesc->HasPath()) {
+ project = clusterDesc->GetPath().GetProject();
+ }
+ YQL_ENSURE(!project.empty(), "Project is not defined. You can define it inside connection, or inside query.");
+
+ if (cluster.empty() && clusterDesc->HasPath()) {
+ cluster = clusterDesc->GetPath().GetCluster();
+ }
+ YQL_ENSURE(!cluster.empty(), "Cluster is not defined. You can define it inside connection, or inside query.");
+
+ auto solomonClusterAtom = Build<TCoAtom>(ctx, shardNode.Pos())
+ .Value(solomonCluster)
+ .Done();
+
+ auto projectAtom = Build<TCoAtom>(ctx, shardNode.Pos())
+ .Value(project)
+ .Done();
+
+ auto clusterAtom = Build<TCoAtom>(ctx, shardNode.Pos())
+ .Value(cluster)
+ .Done();
+
+ auto serviceAtom = Build<TCoAtom>(ctx, shardNode.Pos())
+ .Value(service)
+ .Done();
+
+ auto dqSoShardBuilder = Build<TSoShard>(ctx, shardNode.Pos());
+ dqSoShardBuilder.SolomonCluster(solomonClusterAtom);
+ dqSoShardBuilder.Project(projectAtom);
+ dqSoShardBuilder.Cluster(clusterAtom);
+ dqSoShardBuilder.Service(serviceAtom);
+
+ dqSoShardBuilder.Token<TCoSecureParam>().Name().Build("cluster:default_" + solomonCluster).Build();
+
+ return dqSoShardBuilder.Done();
+ }
+
+private:
+ TSolomonState::TPtr State_;
+};
+
+} // namespace
+
+THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state) {
+ return MakeHolder<TSoPhysicalOptProposalTransformer>(std::move(state));
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp
index 7db0ba0cc69..d84f1502a65 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.cpp
@@ -6,8 +6,8 @@
namespace NYql {
-TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode) {
- return [gateway, supportRtmrMode] (
+TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode) {
+ return [gateway, supportRtmrMode] (
const TString& userName,
const TString& sessionId,
const TGatewaysConfig* gatewaysConfig,
@@ -26,12 +26,12 @@ TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr
auto solomonState = MakeIntrusive<TSolomonState>();
- solomonState->SupportRtmrMode = supportRtmrMode;
+ solomonState->SupportRtmrMode = supportRtmrMode;
solomonState->Types = typeCtx.Get();
solomonState->Gateway = gateway;
- solomonState->DqIntegration = CreateSolomonDqIntegration(solomonState);
+ solomonState->DqIntegration = CreateSolomonDqIntegration(solomonState);
if (gatewaysConfig) {
- solomonState->Configuration->Init(gatewaysConfig->GetSolomon(), typeCtx);
+ solomonState->Configuration->Init(gatewaysConfig->GetSolomon(), typeCtx);
}
TDataProviderInfo info;
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h
index 4055df64a3e..7f4a88a1315 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider.h
@@ -11,21 +11,21 @@ struct TSolomonState : public TThrRefBase
{
using TPtr = TIntrusivePtr<TSolomonState>;
-public:
- bool IsRtmrMode() const {
- return SupportRtmrMode;
- }
-
-public:
- bool SupportRtmrMode = true;
-
+public:
+ bool IsRtmrMode() const {
+ return SupportRtmrMode;
+ }
+
+public:
+ bool SupportRtmrMode = true;
+
ISolomonGateway::TPtr Gateway;
TTypeAnnotationContext* Types = nullptr;
TSolomonConfiguration::TPtr Configuration = MakeIntrusive<TSolomonConfiguration>();
- THolder<IDqIntegration> DqIntegration;
+ THolder<IDqIntegration> DqIntegration;
};
-TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode = true);
+TDataProviderInitializer GetSolomonDataProviderInitializer(ISolomonGateway::TPtr gateway, bool supportRtmrMode = true);
TIntrusivePtr<IDataProvider> CreateSolomonDataSource(TSolomonState::TPtr state);
TIntrusivePtr<IDataProvider> CreateSolomonDataSink(TSolomonState::TPtr state);
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h
index 831bbf98d12..c1f48a5b540 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_provider_impl.h
@@ -19,6 +19,6 @@ THolder<TExecTransformerBase> CreateSolomonDataSourceExecTransformer(TSolomonSta
THolder<TVisitorTransformerBase> CreateSolomonDataSinkTypeAnnotationTransformer(TSolomonState::TPtr state);
THolder<TExecTransformerBase> CreateSolomonDataSinkExecTransformer(TSolomonState::TPtr state);
-THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state);
-
+THolder<IGraphTransformer> CreateSoPhysicalOptProposalTransformer(TSolomonState::TPtr state);
+
} // namespace NYql
diff --git a/ydb/library/yql/providers/solomon/ya.make b/ydb/library/yql/providers/solomon/ya.make
index 54027e73832..579c088007d 100644
--- a/ydb/library/yql/providers/solomon/ya.make
+++ b/ydb/library/yql/providers/solomon/ya.make
@@ -1,7 +1,7 @@
RECURSE(
- async_io
+ async_io
expr_nodes
gateway
- proto
+ proto
provider
)
diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp
index 05976b9dbfe..7ab55cfcfc9 100644
--- a/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp
+++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_read_actor.cpp
@@ -192,7 +192,7 @@ private:
RequestsDone = true;
while(!Blocks.empty())
Blocks.pop();
- Callbacks->OnSourceError(InputIndex, res.GetIssues(), true);
+ Callbacks->OnSourceError(InputIndex, res.GetIssues(), true);
} else {
WakeUpTime = TMonotonic::Now() + Min(TDuration::Seconds(3), TDuration::MilliSeconds(0x30U * (1U << ++Retried)));
ActorSystem->Schedule(WakeUpTime, new IEventHandle(SelfId(), TActorId(), new TEvPrivate::TEvRetryTime));
diff --git a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp
index 0e648486b04..022df277833 100644
--- a/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp
+++ b/ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.cpp
@@ -2,13 +2,13 @@
#include "yql_ydb_read_actor.h"
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h>
-
+
namespace NYql::NDq {
void RegisterYdbReadActorFactory(NYql::NDq::TDqSourceFactory& factory, ::NYdb::TDriver driver, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) {
factory.Register<NYql::NYdb::TSource>("YdbSource",
- [driver, credentialsFactory](NYql::NYdb::TSource&& settings, IDqSourceActorFactory::TArguments&& args) {
- return CreateYdbReadActor(std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, driver, credentialsFactory);
+ [driver, credentialsFactory](NYql::NYdb::TSource&& settings, IDqSourceActorFactory::TArguments&& args) {
+ return CreateYdbReadActor(std::move(settings), args.InputIndex, args.SecureParams, args.TaskParams, args.Callback, driver, credentialsFactory);
});
}
diff --git a/ydb/library/yql/sql/v0/node.cpp b/ydb/library/yql/sql/v0/node.cpp
index 4350b7dfb84..57f0c6a561f 100644
--- a/ydb/library/yql/sql/v0/node.cpp
+++ b/ydb/library/yql/sql/v0/node.cpp
@@ -1459,8 +1459,8 @@ TNodePtr ISource::BuildAggregation(const TString& label) {
BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor),
HoppingWindowSpec->Hop,
HoppingWindowSpec->Interval,
- HoppingWindowSpec->Delay,
- Q("False"));
+ HoppingWindowSpec->Delay,
+ Q("False"));
return Y("Aggregate", label, Q(keysTuple), Q(aggrArgs),
Q(Y(Q(Y(BuildQuotedAtom(Pos, "hopping"), hoppingTraits)))));
diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h
index c916884ecb0..89517574862 100644
--- a/ydb/library/yql/sql/v1/context.h
+++ b/ydb/library/yql/sql/v1/context.h
@@ -246,7 +246,7 @@ namespace NSQLTranslationV1 {
bool PositionalUnionAll = false;
bool BogousStarInGroupByOverJoin = false;
bool UnorderedSubqueries = true;
- bool PragmaDataWatermarks = true;
+ bool PragmaDataWatermarks = true;
bool WarnOnAnsiAliasShadowing = true;
ui32 ResultRowsLimit = 0;
ui64 ResultSizeLimit = 0;
diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp
index d0366be2b03..870bbd9cc7f 100644
--- a/ydb/library/yql/sql/v1/node.cpp
+++ b/ydb/library/yql/sql/v1/node.cpp
@@ -999,7 +999,7 @@ THoppingWindowSpecPtr THoppingWindowSpec::Clone() const {
res->Hop = Hop->Clone();
res->Interval = Interval->Clone();
res->Delay = Delay->Clone();
- res->DataWatermarks = DataWatermarks;
+ res->DataWatermarks = DataWatermarks;
return res;
}
@@ -1780,8 +1780,8 @@ TNodePtr ISource::BuildAggregation(const TString& label) {
BuildLambda(Pos, Y("row"), HoppingWindowSpec->TimeExtractor),
HoppingWindowSpec->Hop,
HoppingWindowSpec->Interval,
- HoppingWindowSpec->Delay,
- HoppingWindowSpec->DataWatermarks ? Q("true") : Q("false"));
+ HoppingWindowSpec->Delay,
+ HoppingWindowSpec->DataWatermarks ? Q("true") : Q("false"));
options = L(options, Q(Y(Q("hopping"), hoppingTraits)));
}
diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h
index 768e5ffc518..de82b45a238 100644
--- a/ydb/library/yql/sql/v1/node.h
+++ b/ydb/library/yql/sql/v1/node.h
@@ -609,7 +609,7 @@ namespace NSQLTranslationV1 {
TNodePtr Hop;
TNodePtr Interval;
TNodePtr Delay;
- bool DataWatermarks;
+ bool DataWatermarks;
TIntrusivePtr<THoppingWindowSpec> Clone() const;
~THoppingWindowSpec() {}
diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp
index 7844bebcb6f..ab5a8647885 100644
--- a/ydb/library/yql/sql/v1/sql.cpp
+++ b/ydb/library/yql/sql/v1/sql.cpp
@@ -7407,7 +7407,7 @@ bool TGroupByClause::HoppingWindow(const TRule_hopping_window_specification& nod
if (!HoppingWindowSpec->Delay) {
return false;
}
- HoppingWindowSpec->DataWatermarks = Ctx.PragmaDataWatermarks;
+ HoppingWindowSpec->DataWatermarks = Ctx.PragmaDataWatermarks;
return true;
}
@@ -9574,22 +9574,22 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
} else if (normalizedPragma == "disableunorderedsubqueries") {
Ctx.UnorderedSubqueries = false;
Ctx.IncrementMonCounter("sql_pragma", "DisableUnorderedSubqueries");
- } else if (normalizedPragma == "datawatermarks") {
- if (values.size() != 1 || !values[0].GetLiteral()
- || ! (*values[0].GetLiteral() == "enable" || *values[0].GetLiteral() == "disable"))
- {
- Error() << "Expected `enable|disable' argument for: " << pragma;
- Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
- }
-
- if (*values[0].GetLiteral() == "enable") {
- Ctx.PragmaDataWatermarks = true;
- } else if (*values[0].GetLiteral() == "disable") {
- Ctx.PragmaDataWatermarks = false;
- }
-
- Ctx.IncrementMonCounter("sql_pragma", "DataWatermarks");
+ } else if (normalizedPragma == "datawatermarks") {
+ if (values.size() != 1 || !values[0].GetLiteral()
+ || ! (*values[0].GetLiteral() == "enable" || *values[0].GetLiteral() == "disable"))
+ {
+ Error() << "Expected `enable|disable' argument for: " << pragma;
+ Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
+ return {};
+ }
+
+ if (*values[0].GetLiteral() == "enable") {
+ Ctx.PragmaDataWatermarks = true;
+ } else if (*values[0].GetLiteral() == "disable") {
+ Ctx.PragmaDataWatermarks = false;
+ }
+
+ Ctx.IncrementMonCounter("sql_pragma", "DataWatermarks");
} else if (normalizedPragma == "flexibletypes") {
Ctx.FlexibleTypes = true;
Ctx.IncrementMonCounter("sql_pragma", "FlexibleTypes");
diff --git a/ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp b/ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp
index d4da1071bca..a83941f90dd 100644
--- a/ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp
+++ b/ydb/library/yql/udfs/common/clickhouse/client/src/IO/ReadHelpers.cpp
@@ -1010,7 +1010,7 @@ void skipJSONField(ReadBuffer & buf, const StringRef & name_of_field)
}
else
{
- throw Exception("Unexpected symbol '" + std::string(buf.position(), 1) + "' for key '" + name_of_field.toString() + "'", ErrorCodes::INCORRECT_DATA);
+ throw Exception("Unexpected symbol '" + std::string(buf.position(), 1) + "' for key '" + name_of_field.toString() + "'", ErrorCodes::INCORRECT_DATA);
}
}
diff --git a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
index 8157de7cf4b..1e6eac30556 100644
--- a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
+++ b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
@@ -61,7 +61,7 @@ namespace ErrorCodes
extern const int UNKNOWN_TYPE;
extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE;
extern const int THERE_IS_NO_COLUMN;
- extern const int DUPLICATE_COLUMN;
+ extern const int DUPLICATE_COLUMN;
extern const int BAD_ARGUMENTS;
extern const int UNKNOWN_EXCEPTION;
}
@@ -529,9 +529,9 @@ void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr<arr
for (const auto& column_name : table->ColumnNames())
{
std::shared_ptr<arrow::ChunkedArray> arrow_column = table->GetColumnByName(column_name);
- if (!arrow_column) {
- throw Exception{ErrorCodes::DUPLICATE_COLUMN, "Column '{}' is duplicated.", column_name};
- }
+ if (!arrow_column) {
+ throw Exception{ErrorCodes::DUPLICATE_COLUMN, "Column '{}' is duplicated.", column_name};
+ }
name_to_column_ptr[column_name] = arrow_column;
}
diff --git a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp
index 2e9776abf5c..927f612f27f 100644
--- a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp
+++ b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp
@@ -16,7 +16,7 @@ namespace ErrorCodes
RawBLOBRowInputFormat::RawBLOBRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_)
: IRowInputFormat(header_, in_, std::move(params_))
{
- if (header_.columns() > 1 || header_.columns() == 0)
+ if (header_.columns() > 1 || header_.columns() == 0)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"This input format is only suitable for tables with a single column of type String but the number of columns is {}",
header_.columns());
diff --git a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp
index bf832960b90..85937935f11 100644
--- a/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp
+++ b/ydb/library/yql/udfs/common/clickhouse/client/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp
@@ -52,7 +52,7 @@ static bool readName(ReadBuffer & buf, StringRef & ref, String & tmp)
if (next_pos == buf.buffer().end())
{
tmp.append(buf.position(), next_pos - buf.position());
- buf.position() = buf.buffer().end();
+ buf.position() = buf.buffer().end();
buf.next();
continue;
}
diff --git a/ydb/library/yql/udfs/common/clickhouse/client/ya.make b/ydb/library/yql/udfs/common/clickhouse/client/ya.make
index 106a6d0f85c..d9ef0a66458 100644
--- a/ydb/library/yql/udfs/common/clickhouse/client/ya.make
+++ b/ydb/library/yql/udfs/common/clickhouse/client/ya.make
@@ -419,7 +419,7 @@ IF (OS_LINUX AND CLANG AND NOT WITH_VALGRIND)
)
CFLAGS (
- -DARCADIA_BUILD -DOS_LINUX -DUSE_ARROW=0 -DUSE_PARQUET=0 -DUSE_ORC=0 -DUSE_AVRO=0 -DUSE_UNWIND=0 -DDBMS_VERSION_MAJOR=21 -DDBMS_VERSION_MINOR=18 -DDBMS_VERSION_PATCH=0
+ -DARCADIA_BUILD -DOS_LINUX -DUSE_ARROW=0 -DUSE_PARQUET=0 -DUSE_ORC=0 -DUSE_AVRO=0 -DUSE_UNWIND=0 -DDBMS_VERSION_MAJOR=21 -DDBMS_VERSION_MINOR=18 -DDBMS_VERSION_PATCH=0
-Wno-unused-parameter
)
diff --git a/ydb/public/api/protos/yq.proto b/ydb/public/api/protos/yq.proto
index ad9a13eae82..6aa7169a890 100644
--- a/ydb/public/api/protos/yq.proto
+++ b/ydb/public/api/protos/yq.proto
@@ -58,7 +58,7 @@ enum ExecuteMode {
enum QueryAction {
QUERY_ACTION_UNSPECIFIED = 0;
PAUSE = 1; // Pause the query, with the possibility of its quick resumption
- PAUSE_GRACEFULLY = 2; // Similar to PAUSE, only suspends the query allowing it to pause in checkpoint. Can work for a long time
+ PAUSE_GRACEFULLY = 2; // Similar to PAUSE, only suspends the query allowing it to pause in checkpoint. Can work for a long time
ABORT = 3; // Stop the query
ABORT_GRACEFULLY = 4; // Similar to ABORT, only stops the query in checkpoint
RESUME = 5; // Resumes the execution of the query. Works only for PAUSE queries
@@ -66,7 +66,7 @@ enum QueryAction {
enum StateLoadMode {
STATE_LOAD_MODE_UNSPECIFIED = 0;
- EMPTY = 1; // Start the query with an empty state
+ EMPTY = 1; // Start the query with an empty state
FROM_LAST_CHECKPOINT = 2; // Start the query with the state that is saved in the last checkpoint
}
@@ -89,8 +89,8 @@ message StreamingDisposition {
}
oneof disposition {
- google.protobuf.Empty oldest = 1; // Start processing with the oldest offset
- google.protobuf.Empty fresh = 2; // Start processing with the fresh offset
+ google.protobuf.Empty oldest = 1; // Start processing with the oldest offset
+ google.protobuf.Empty fresh = 2; // Start processing with the fresh offset
FromTime from_time = 3; // Start processing with offset from the specified time
TimeAgo time_ago = 4; // Start processing with offset some time ago
FromLastCheckpoint from_last_checkpoint = 5; // Start processing with offset which corresponds to the last checkpoint
@@ -418,12 +418,12 @@ message DataStreams {
bool secure = 5;
}
-message Monitoring {
- string project = 1 [(Ydb.length).le = 200];
- string cluster = 2 [(Ydb.length).le = 200];
- IamAuth auth = 3;
-}
-
+message Monitoring {
+ string project = 1 [(Ydb.length).le = 200];
+ string cluster = 2 [(Ydb.length).le = 200];
+ IamAuth auth = 3;
+}
+
message YdbDatabase {
string database_id = 1 [(Ydb.length).le = 1024];
IamAuth auth = 2;
@@ -466,7 +466,7 @@ message ConnectionSetting {
ClickHouseCluster clickhouse_cluster = 2;
DataStreams data_streams = 3;
ObjectStorageConnection object_storage = 4;
- Monitoring monitoring = 5;
+ Monitoring monitoring = 5;
}
}
diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h
index 3927916c0ac..c067928c785 100644
--- a/ydb/services/ydb/ydb_common_ut.h
+++ b/ydb/services/ydb/ydb_common_ut.h
@@ -12,8 +12,8 @@
#include <contrib/libs/apache/arrow/cpp/src/arrow/csv/api.h> // for WriteCSV()
#include <contrib/libs/apache/arrow/cpp/src/arrow/io/api.h>
-#include <functional>
-
+#include <functional>
+
using namespace NKikimr;
namespace NYdb {
@@ -47,8 +47,8 @@ public:
const TVector<NKikimrKqp::TKqpSetting>& kqpSettings = {},
TAutoPtr<TLogBackend> logBackend = {},
bool enableYq = false,
- TAppPrepare::TFnReg udfFrFactory = nullptr,
- std::function<void(TServerSettings& settings)> builder = nullptr)
+ TAppPrepare::TFnReg udfFrFactory = nullptr,
+ std::function<void(TServerSettings& settings)> builder = nullptr)
{
ui16 port = PortManager.GetPort(2134);
ui16 grpc = PortManager.GetPort(2135);
@@ -69,7 +69,7 @@ public:
ServerSettings->AddStoragePoolType("hdd2");
}
ServerSettings->SetAppConfig(appConfig);
- ServerSettings->AuthConfig = appConfig.GetAuthConfig();
+ ServerSettings->AuthConfig = appConfig.GetAuthConfig();
ServerSettings->FeatureFlags = appConfig.GetFeatureFlags();
ServerSettings->SetKqpSettings(kqpSettings);
ServerSettings->SetEnableAsyncIndexes(true);
@@ -87,9 +87,9 @@ public:
if (udfFrFactory) {
ServerSettings->SetFrFactory(udfFrFactory);
}
- if (builder) {
- builder(*ServerSettings);;
- }
+ if (builder) {
+ builder(*ServerSettings);;
+ }
Server_.Reset(new TServer(*ServerSettings));
Tenants_.Reset(new Tests::TTenants(Server_));