diff options
author | robot-piglet <robot-piglet@yandex-team.com> | 2024-03-01 17:42:44 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2024-03-01 17:53:54 +0300 |
commit | 280ea5e89bc75e2f23148f30347dbd2795d160a5 (patch) | |
tree | 959129f5896a995afd002865673f73bde9fecff6 | |
parent | c90958a82ed1e508a3db6b0e75e682ae8f3f5659 (diff) | |
download | ydb-280ea5e89bc75e2f23148f30347dbd2795d160a5.tar.gz |
Intermediate changes
-rw-r--r-- | yt/yt/client/api/query_tracker_client.h | 4 | ||||
-rw-r--r-- | yt/yt/client/api/rpc_proxy/api_service_proxy.h | 10 | ||||
-rw-r--r-- | yt/yt/client/api/rpc_proxy/client_impl.cpp | 227 | ||||
-rw-r--r-- | yt/yt/client/api/rpc_proxy/helpers.cpp | 222 | ||||
-rw-r--r-- | yt/yt/client/api/rpc_proxy/helpers.h | 19 | ||||
-rw-r--r-- | yt/yt/client/table_client/config.cpp | 3 | ||||
-rw-r--r-- | yt/yt/core/ytree/unittests/yson_struct_ut.cpp | 166 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct.cpp | 4 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct.h | 2 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct_detail-inl.h | 421 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct_detail.cpp | 3 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct_detail.h | 25 | ||||
-rw-r--r-- | yt/yt/core/ytree/yson_struct_enum.h | 8 | ||||
-rw-r--r-- | yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto | 203 |
14 files changed, 962 insertions, 355 deletions
diff --git a/yt/yt/client/api/query_tracker_client.h b/yt/yt/client/api/query_tracker_client.h index f85a6757d2..e2cb81d27e 100644 --- a/yt/yt/client/api/query_tracker_client.h +++ b/yt/yt/client/api/query_tracker_client.h @@ -15,7 +15,7 @@ struct TQueryTrackerOptions TString QueryTrackerStage = "production"; }; -DEFINE_ENUM(ContentType, +DEFINE_ENUM(EContentType, ((RawInlineData) (0)) ((Url) (1)) ); @@ -25,7 +25,7 @@ struct TQueryFile { TString Name; TString Content; - ContentType Type; + EContentType Type; REGISTER_YSON_STRUCT(TQueryFile); diff --git a/yt/yt/client/api/rpc_proxy/api_service_proxy.h b/yt/yt/client/api/rpc_proxy/api_service_proxy.h index 6aaf2c1e36..d90e1ede0a 100644 --- a/yt/yt/client/api/rpc_proxy/api_service_proxy.h +++ b/yt/yt/client/api/rpc_proxy/api_service_proxy.h @@ -183,6 +183,16 @@ public: DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, PausePipeline); DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, GetPipelineStatus); + // Query tracker + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, StartQuery); + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, AbortQuery); + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, GetQueryResult); + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, ReadQueryResult); + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, GetQuery); + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, ListQueries); + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, AlterQuery); + DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, GetQueryTrackerInfo); + // Misc DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, CheckClusterLiveness); }; diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index 1089d99179..6daa563030 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -2019,60 +2019,237 @@ TFuture<TRequestRestartResult> TClient::RequestRestart( } TFuture<NQueryTrackerClient::TQueryId> TClient::StartQuery( - NQueryTrackerClient::EQueryEngine /*engine*/, - const TString& /*query*/, - const TStartQueryOptions& /*options*/) + NQueryTrackerClient::EQueryEngine engine, + const TString& query, + const TStartQueryOptions& options) { - ThrowUnimplemented("StartQuery"); + auto proxy = CreateApiServiceProxy(); + + auto req = proxy.StartQuery(); + SetTimeoutOptions(*req, options); + + req->set_query_tracker_stage(options.QueryTrackerStage); + req->set_engine(NProto::ConvertQueryEngineToProto(engine)); + req->set_query(query); + req->set_draft(options.Draft); + + if (options.Settings) { + req->set_settings(ConvertToYsonString(options.Settings).ToString()); + } + if (options.Annotations) { + req->set_annotations(ConvertToYsonString(options.Annotations).ToString()); + } + if (options.AccessControlObject) { + req->set_access_control_object(*options.AccessControlObject); + } + + for (const auto& file : options.Files) { + auto* protoFile = req->add_files(); + protoFile->set_name(file->Name); + protoFile->set_content(file->Content); + protoFile->set_type(static_cast<NProto::EContentType>(file->Type)); + } + + return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspStartQueryPtr& rsp) { + return FromProto<NQueryTrackerClient::TQueryId>(rsp->query_id()); + })); } TFuture<void> TClient::AbortQuery( - NQueryTrackerClient::TQueryId /*queryId*/, - const TAbortQueryOptions& /*options*/) + NQueryTrackerClient::TQueryId queryId, + const TAbortQueryOptions& options) { - ThrowUnimplemented("AbortQuery"); + auto proxy = CreateApiServiceProxy(); + + auto req = proxy.AbortQuery(); + SetTimeoutOptions(*req, options); + + req->set_query_tracker_stage(options.QueryTrackerStage); + ToProto(req->mutable_query_id(), queryId); + + if (options.AbortMessage) { + req->set_abort_message(*options.AbortMessage); + } + + return req->Invoke().AsVoid(); } TFuture<TQueryResult> TClient::GetQueryResult( - NQueryTrackerClient::TQueryId /*queryId*/, - i64 /*resultIndex*/, - const TGetQueryResultOptions& /*options*/) + NQueryTrackerClient::TQueryId queryId, + i64 resultIndex, + const TGetQueryResultOptions& options) { - ThrowUnimplemented("GetQueryResult"); + auto proxy = CreateApiServiceProxy(); + + auto req = proxy.GetQueryResult(); + SetTimeoutOptions(*req, options); + + req->set_query_tracker_stage(options.QueryTrackerStage); + ToProto(req->mutable_query_id(), queryId); + req->set_result_index(resultIndex); + + return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetQueryResultPtr& rsp) { + return TQueryResult{ + .Id = FromProto<NQueryTrackerClient::TQueryId>(rsp->query_id()), + .ResultIndex = rsp->result_index(), + .Error = FromProto<TError>(rsp->error()), + .Schema = rsp->has_schema() ? FromProto<NTableClient::TTableSchemaPtr>(rsp->schema()) : nullptr, + .DataStatistics = FromProto<NChunkClient::NProto::TDataStatistics>(rsp->data_statistics()), + .IsTruncated = rsp->is_truncated(), + }; + })); } TFuture<IUnversionedRowsetPtr> TClient::ReadQueryResult( - NQueryTrackerClient::TQueryId /*queryId*/, - i64 /*resultIndex*/, - const TReadQueryResultOptions& /*options*/) + NQueryTrackerClient::TQueryId queryId, + i64 resultIndex, + const TReadQueryResultOptions& options) { - ThrowUnimplemented("AbortQuery"); + auto proxy = CreateApiServiceProxy(); + + auto req = proxy.ReadQueryResult(); + SetTimeoutOptions(*req, options); + + req->set_query_tracker_stage(options.QueryTrackerStage); + ToProto(req->mutable_query_id(), queryId); + req->set_result_index(resultIndex); + + if (options.Columns) { + auto* protoColumns = req->mutable_columns(); + for (const auto& column : *options.Columns) { + protoColumns->add_items(column); + } + } + if (options.LowerRowIndex) { + req->set_lower_row_index(*options.LowerRowIndex); + } + if (options.UpperRowIndex) { + req->set_upper_row_index(*options.UpperRowIndex); + } + + return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspReadQueryResultPtr& rsp) { + return DeserializeRowset<TUnversionedRow>( + rsp->rowset_descriptor(), + MergeRefsToRef<TRpcProxyClientBufferTag>(rsp->Attachments())); + })); } TFuture<TQuery> TClient::GetQuery( - NQueryTrackerClient::TQueryId /*queryId*/, - const TGetQueryOptions& /*options*/) + NQueryTrackerClient::TQueryId queryId, + const TGetQueryOptions& options) { - ThrowUnimplemented("GetQuery"); + auto proxy = CreateApiServiceProxy(); + + auto req = proxy.GetQuery(); + SetTimeoutOptions(*req, options); + + req->set_query_tracker_stage(options.QueryTrackerStage); + ToProto(req->mutable_query_id(), queryId); + + if (options.Attributes) { + ToProto(req->mutable_attributes(), options.Attributes); + } + if (options.Timestamp) { + req->set_timestamp(ToProto<i64>(options.Timestamp)); + } + + return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetQueryPtr& rsp) { + return FromProto<TQuery>(rsp->query()); + })); } TFuture<TListQueriesResult> TClient::ListQueries( - const TListQueriesOptions& /*options*/) + const TListQueriesOptions& options) { - ThrowUnimplemented("ListQueries"); + auto proxy = CreateApiServiceProxy(); + + auto req = proxy.ListQueries(); + SetTimeoutOptions(*req, options); + + req->set_query_tracker_stage(options.QueryTrackerStage); + + if (options.FromTime) { + req->set_from_time(NYT::ToProto<i64>(*options.FromTime)); + } + if (options.ToTime) { + req->set_to_time(NYT::ToProto<i64>(*options.ToTime)); + } + if (options.CursorTime) { + req->set_cursor_time(NYT::ToProto<i64>(*options.CursorTime)); + } + req->set_cursor_direction(static_cast<NProto::EOperationSortDirection>(options.CursorDirection)); + + if (options.UserFilter) { + req->set_user_filter(*options.UserFilter); + } + if (options.StateFilter) { + req->set_state_filter(NProto::ConvertQueryStateToProto(*options.StateFilter)); + } + if (options.EngineFilter) { + req->set_engine_filter(NProto::ConvertQueryEngineToProto(*options.EngineFilter)); + } + if (options.SubstrFilter) { + req->set_substr_filter(*options.SubstrFilter); + } + + req->set_limit(options.Limit); + + if (options.Attributes) { + ToProto(req->mutable_attributes(), options.Attributes); + } + + return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspListQueriesPtr& rsp) { + return TListQueriesResult{ + .Queries = FromProto<std::vector<TQuery>>(rsp->queries()), + .Incomplete = rsp->incomplete(), + .Timestamp = rsp->timestamp(), + }; + })); } TFuture<void> TClient::AlterQuery( - NQueryTrackerClient::TQueryId /*queryId*/, - const TAlterQueryOptions& /*options*/) + NQueryTrackerClient::TQueryId queryId, + const TAlterQueryOptions& options) { - ThrowUnimplemented("AlterQuery"); + auto proxy = CreateApiServiceProxy(); + + auto req = proxy.AlterQuery(); + SetTimeoutOptions(*req, options); + + req->set_query_tracker_stage(options.QueryTrackerStage); + ToProto(req->mutable_query_id(), queryId); + + if (options.Annotations) { + req->set_annotations(ConvertToYsonString(options.Annotations).ToString()); + } + if (options.AccessControlObject) { + req->set_access_control_object(*options.AccessControlObject); + } + + return req->Invoke().AsVoid(); } TFuture<TGetQueryTrackerInfoResult> TClient::GetQueryTrackerInfo( - const TGetQueryTrackerInfoOptions& /*options*/) + const TGetQueryTrackerInfoOptions& options) { - ThrowUnimplemented("GetQueryTrackerInfo"); + auto proxy = CreateApiServiceProxy(); + + auto req = proxy.GetQueryTrackerInfo(); + SetTimeoutOptions(*req, options); + + req->set_query_tracker_stage(options.QueryTrackerStage); + + if (options.Attributes) { + ToProto(req->mutable_attributes(), options.Attributes); + } + + return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetQueryTrackerInfoPtr& rsp) { + return TGetQueryTrackerInfoResult{ + .ClusterName = rsp->cluster_name(), + .SupportedFeatures = TYsonString(rsp->supported_features()), + .AccessControlObjects = FromProto<std::vector<TString>>(rsp->access_control_objects()), + }; + })); } TFuture<NBundleControllerClient::TBundleConfigDescriptorPtr> TClient::GetBundleConfig( diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp index fdbf14614a..1dff960561 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.cpp +++ b/yt/yt/client/api/rpc_proxy/helpers.cpp @@ -1332,6 +1332,136 @@ void FromProto( FromProto(&manifest->Clusters, protoManifest.clusters()); } +void ToProto( + NProto::TQuery* protoQuery, + const NApi::TQuery& query) +{ + protoQuery->Clear(); + + ToProto(protoQuery->mutable_id(), query.Id); + + if (query.Engine) { + protoQuery->set_engine(ConvertQueryEngineToProto(*query.Engine)); + } + if (query.Query) { + protoQuery->set_query(*query.Query); + } + if (query.Files) { + protoQuery->set_files(query.Files->ToString()); + } + if (query.StartTime) { + protoQuery->set_start_time(NYT::ToProto<i64>(*query.StartTime)); + } + if (query.FinishTime) { + protoQuery->set_start_time(NYT::ToProto<i64>(*query.FinishTime)); + } + if (query.Settings) { + protoQuery->set_settings(query.Settings.ToString()); + } + if (query.User) { + protoQuery->set_user(*query.User); + } + if (query.AccessControlObject) { + protoQuery->set_access_control_object(*query.AccessControlObject); + } + if (query.State) { + protoQuery->set_state(ConvertQueryStateToProto(*query.State)); + } + if (query.ResultCount) { + protoQuery->set_result_count(*query.ResultCount); + } + if (query.Progress) { + protoQuery->set_progress(query.Progress.ToString()); + } + if (query.Error) { + ToProto(protoQuery->mutable_error(), *query.Error); + } + if (query.Annotations) { + protoQuery->set_annotations(query.Annotations.ToString()); + } + if (query.OtherAttributes) { + ToProto(protoQuery->mutable_other_attributes(), *query.OtherAttributes); + } +} + +void FromProto( + NApi::TQuery* query, + const NProto::TQuery& protoQuery) +{ + FromProto(&query->Id, protoQuery.id()); + + if (protoQuery.has_engine()) { + query->Engine = ConvertQueryEngineFromProto(protoQuery.engine()); + } else { + query->Engine.reset(); + } + if (protoQuery.has_query()) { + query->Query = protoQuery.query(); + } else { + query->Query.reset(); + } + if (protoQuery.has_files()) { + query->Files = TYsonString(protoQuery.files()); + } else { + query->Files.reset(); + } + if (protoQuery.has_start_time()) { + query->StartTime = TInstant::FromValue(protoQuery.start_time()); + } else { + query->StartTime.reset(); + } + if (protoQuery.has_finish_time()) { + query->FinishTime = TInstant::FromValue(protoQuery.finish_time()); + } else { + query->FinishTime.reset(); + } + if (protoQuery.has_settings()) { + query->Settings = TYsonString(protoQuery.settings()); + } else { + query->Settings = TYsonString{}; + } + if (protoQuery.has_user()) { + query->User = protoQuery.user(); + } else { + query->User.reset(); + } + if (protoQuery.has_access_control_object()) { + query->AccessControlObject = protoQuery.access_control_object(); + } else { + query->AccessControlObject.reset(); + } + if (protoQuery.has_state()) { + query->State = ConvertQueryStateFromProto(protoQuery.state()); + } else { + query->State.reset(); + } + if (protoQuery.result_count()) { + query->ResultCount = protoQuery.result_count(); + } else { + query->ResultCount.reset(); + } + if (protoQuery.has_progress()) { + query->Progress = TYsonString(protoQuery.progress()); + } else { + query->Progress = TYsonString{}; + } + if (protoQuery.has_error()) { + query->Error = FromProto<TError>(protoQuery.error()); + } else { + query->Error.reset(); + } + if (protoQuery.has_annotations()) { + query->Annotations = TYsonString(protoQuery.annotations()); + } else { + query->Annotations = TYsonString{}; + } + if (protoQuery.has_other_attributes()) { + query->OtherAttributes = NYTree::FromProto(protoQuery.other_attributes()); + } else if (query->OtherAttributes) { + query->OtherAttributes->Clear(); + } +} + //////////////////////////////////////////////////////////////////////////////// // ENUMS //////////////////////////////////////////////////////////////////////////////// @@ -1648,6 +1778,98 @@ NJobTrackerClient::EJobState ConvertJobStateFromProto( YT_ABORT(); } +NProto::EQueryEngine ConvertQueryEngineToProto( + NQueryTrackerClient::EQueryEngine queryEngine) +{ + switch (queryEngine) { + case NQueryTrackerClient::EQueryEngine::Ql: + return NProto::EQueryEngine::QE_QL; + case NQueryTrackerClient::EQueryEngine::Yql: + return NProto::EQueryEngine::QE_YQL; + case NQueryTrackerClient::EQueryEngine::Chyt: + return NProto::EQueryEngine::QE_CHYT; + case NQueryTrackerClient::EQueryEngine::Mock: + return NProto::EQueryEngine::QE_MOCK; + case NQueryTrackerClient::EQueryEngine::Spyt: + return NProto::EQueryEngine::QE_SPYT; + } + YT_ABORT(); +} + +NQueryTrackerClient::EQueryEngine ConvertQueryEngineFromProto( + NProto::EQueryEngine proto) +{ + switch (proto) { + case NProto::EQueryEngine::QE_QL: + return NQueryTrackerClient::EQueryEngine::Ql; + case NProto::EQueryEngine::QE_YQL: + return NQueryTrackerClient::EQueryEngine::Yql; + case NProto::EQueryEngine::QE_CHYT: + return NQueryTrackerClient::EQueryEngine::Chyt; + case NProto::EQueryEngine::QE_MOCK: + return NQueryTrackerClient::EQueryEngine::Mock; + case NProto::EQueryEngine::QE_SPYT: + return NQueryTrackerClient::EQueryEngine::Spyt; + case NProto::EQueryEngine::QE_UNKNOWN: + THROW_ERROR_EXCEPTION("Protobuf contains unknown value for query engine"); + } + YT_ABORT(); +} + +NProto::EQueryState ConvertQueryStateToProto( + NQueryTrackerClient::EQueryState queryState) +{ + switch (queryState) { + case NQueryTrackerClient::EQueryState::Draft: + return NProto::EQueryState::QS_DRAFT; + case NQueryTrackerClient::EQueryState::Pending: + return NProto::EQueryState::QS_PENDING; + case NQueryTrackerClient::EQueryState::Running: + return NProto::EQueryState::QS_RUNNING; + case NQueryTrackerClient::EQueryState::Aborting: + return NProto::EQueryState::QS_ABORTING; + case NQueryTrackerClient::EQueryState::Aborted: + return NProto::EQueryState::QS_ABORTED; + case NQueryTrackerClient::EQueryState::Completing: + return NProto::EQueryState::QS_COMPLETING; + case NQueryTrackerClient::EQueryState::Completed: + return NProto::EQueryState::QS_COMPLETED; + case NQueryTrackerClient::EQueryState::Failing: + return NProto::EQueryState::QS_FAILING; + case NQueryTrackerClient::EQueryState::Failed: + return NProto::EQueryState::QS_FAILED; + } + YT_ABORT(); +} + +NQueryTrackerClient::EQueryState ConvertQueryStateFromProto( + NProto::EQueryState proto) +{ + switch (proto) { + case NProto::EQueryState::QS_DRAFT: + return NQueryTrackerClient::EQueryState::Draft; + case NProto::EQueryState::QS_PENDING: + return NQueryTrackerClient::EQueryState::Pending; + case NProto::EQueryState::QS_RUNNING: + return NQueryTrackerClient::EQueryState::Running; + case NProto::EQueryState::QS_ABORTING: + return NQueryTrackerClient::EQueryState::Aborting; + case NProto::EQueryState::QS_ABORTED: + return NQueryTrackerClient::EQueryState::Aborted; + case NProto::EQueryState::QS_COMPLETING: + return NQueryTrackerClient::EQueryState::Completing; + case NProto::EQueryState::QS_COMPLETED: + return NQueryTrackerClient::EQueryState::Completed; + case NProto::EQueryState::QS_FAILING: + return NQueryTrackerClient::EQueryState::Failing; + case NProto::EQueryState::QS_FAILED: + return NQueryTrackerClient::EQueryState::Failed; + case NProto::EQueryState::QS_UNKNOWN: + THROW_ERROR_EXCEPTION("Protobuf contains unknown value for query state"); + } + YT_ABORT(); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace NProto diff --git a/yt/yt/client/api/rpc_proxy/helpers.h b/yt/yt/client/api/rpc_proxy/helpers.h index 4b01c4ae11..722e47380e 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.h +++ b/yt/yt/client/api/rpc_proxy/helpers.h @@ -228,6 +228,14 @@ void FromProto( NApi::TBackupManifest* manifest, const NProto::TBackupManifest& protoManifest); +void ToProto( + NProto::TQuery* protoQuery, + const NApi::TQuery& query); + +void FromProto( + NApi::TQuery* query, + const NProto::TQuery& protoQuery); + NProto::EOperationType ConvertOperationTypeToProto( NScheduler::EOperationType operationType); @@ -252,6 +260,17 @@ NProto::EJobState ConvertJobStateToProto( NJobTrackerClient::EJobState ConvertJobStateFromProto( NProto::EJobState proto); +NProto::EQueryEngine ConvertQueryEngineToProto( + NQueryTrackerClient::EQueryEngine queryEngine); + +NQueryTrackerClient::EQueryEngine ConvertQueryEngineFromProto( + NProto::EQueryEngine proto); + +NProto::EQueryState ConvertQueryStateToProto( + NQueryTrackerClient::EQueryState queryState); + +NQueryTrackerClient::EQueryState ConvertQueryStateFromProto( + NProto::EQueryState proto); } // namespace NProto //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/table_client/config.cpp b/yt/yt/client/table_client/config.cpp index 87373d5b95..d8629254d5 100644 --- a/yt/yt/client/table_client/config.cpp +++ b/yt/yt/client/table_client/config.cpp @@ -277,7 +277,8 @@ void TDictionaryCompressionConfig::Register(TRegistrar registrar) .Default({ EDictionaryCompressionPolicy::LargeChunkFirst, EDictionaryCompressionPolicy::FreshChunkFirst, - }); + }) + .ResetOnLoad(); registrar.Parameter("policy_probation_samples_size", &TThis::PolicyProbationSamplesSize) .GreaterThan(0) diff --git a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp index 71457da7aa..1410442a1a 100644 --- a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp +++ b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp @@ -602,12 +602,12 @@ TEST(TYsonStructTest, LoadSingleParameter) auto config = New<TTestConfig>(); config->NullableInt = 10; - config->LoadParameter("my_string", ConvertToNode("test"), EMergeStrategy::Default); + config->LoadParameter("my_string", ConvertToNode("test")); EXPECT_EQ("test", config->MyString); EXPECT_EQ(10, config->NullableInt); } -TEST(TYsonStructTest, LoadSingleParameterWithMergeStrategy) +TEST(TYsonStructTest, LoadSingleParameterOverwriteDefaults) { auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory()); builder->BeginTree(); @@ -619,15 +619,9 @@ TEST(TYsonStructTest, LoadSingleParameterWithMergeStrategy) auto config1 = New<TTestConfig>(); config1->Subconfig->MyBool = true; - config1->LoadParameter("sub", subConfig, EMergeStrategy::Default); + config1->LoadParameter("sub", subConfig); EXPECT_EQ(100, config1->Subconfig->MyInt); - EXPECT_TRUE(config1->Subconfig->MyBool); // Subconfig merged by default. - - auto config2 = New<TTestConfig>(); - config2->Subconfig->MyBool = true; - config2->LoadParameter("sub", subConfig, EMergeStrategy::Overwrite); - EXPECT_EQ(100, config2->Subconfig->MyInt); - EXPECT_FALSE(config2->Subconfig->MyBool); // Overwrite destroyed previous values. + EXPECT_FALSE(config1->Subconfig->MyBool); // Subconfig is overwritten. } TEST(TYsonStructTest, ResetSingleParameter) @@ -993,11 +987,11 @@ TEST(TYsonStructTest, EnumAsKeyToYHash) }; TString serialized = "{\"value0\"=\"abc\";}"; - ASSERT_EQ(serialized, ConvertToYsonString(original, EYsonFormat::Text).AsStringBuf()); + EXPECT_EQ(serialized, ConvertToYsonString(original, EYsonFormat::Text).AsStringBuf()); Deserialize(deserialized, ConvertToNode(TYsonString(serialized, EYsonType::Node))); - ASSERT_EQ(original, deserialized); + EXPECT_EQ(original, deserialized); } //////////////////////////////////////////////////////////////////////////////// @@ -1998,8 +1992,7 @@ public: sub.MyInt = 11; sub.MyString = "x"; return sub; - }) - .MergeBy(EMergeStrategy::Combine); + }); } }; @@ -2096,5 +2089,150 @@ TEST(TYsonStructTest, CustomSubExternalizedStruct) //////////////////////////////////////////////////////////////////////////////// +TIntrusivePtr<TSimpleYsonStruct> CreateSimpleYsonStruct(int value) +{ + auto result = New<TSimpleYsonStruct>(); + result->IntValue = value; + return result; +} + +class TTestingNestedListWithCustomDefault + : public TYsonStruct +{ +public: + std::vector<TIntrusivePtr<TSimpleYsonStruct>> NestedList; + + REGISTER_YSON_STRUCT(TTestingNestedListWithCustomDefault); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("nested_list_1", &TThis::NestedList) + .DefaultCtor([] { + return std::vector{CreateSimpleYsonStruct(5)}; + }); + } +}; + +TEST(TYsonStructTest, NestedListWithCustomDefault) +{ + { + auto testInput = TYsonString(TStringBuf("{}")); + auto deserialized = ConvertTo<TIntrusivePtr<TTestingNestedListWithCustomDefault>>(testInput); + + EXPECT_EQ(deserialized->NestedList.size(), 1u); + EXPECT_EQ(deserialized->NestedList[0]->IntValue, 5); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TTestingNestedMapWithCustomDefault + : public TYsonStruct +{ +public: + THashMap<TString, TIntrusivePtr<TSimpleYsonStruct>> NestedMap; + + REGISTER_YSON_STRUCT(TTestingNestedMapWithCustomDefault); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("nested_map", &TThis::NestedMap) + .DefaultCtor([] { + return THashMap<TString, TIntrusivePtr<TSimpleYsonStruct>>{ + {"foo", CreateSimpleYsonStruct(42)}, + {"bar", CreateSimpleYsonStruct(7)}, + }; + }); + } +}; + +TEST(TYsonStructTest, NestedMapWithCustomDefault) +{ + { + auto testInput = TYsonString(TStringBuf("{}")); + auto deserialized = ConvertTo<TIntrusivePtr<TTestingNestedMapWithCustomDefault>>(testInput); + + EXPECT_EQ(deserialized->NestedMap.size(), 2u); + EXPECT_EQ(deserialized->NestedMap["foo"]->IntValue, 42); + EXPECT_EQ(deserialized->NestedMap["bar"]->IntValue, 7); + + auto testNode = BuildYsonNodeFluently() + .BeginMap() + .Item("nested_map") + .BeginMap() + .Item("baz") + .BeginMap() + .Item("int_value").Value(33) + .EndMap() + .Item("foo") + .BeginMap() + .Item("int_value").Value(88) + .EndMap() + .EndMap() + .EndMap(); + Deserialize(deserialized, testNode->AsMap()); + EXPECT_EQ(deserialized->NestedMap.size(), 3u); + EXPECT_EQ(deserialized->NestedMap["baz"]->IntValue, 33); + EXPECT_EQ(deserialized->NestedMap["foo"]->IntValue, 88); + EXPECT_EQ(deserialized->NestedMap["bar"]->IntValue, 7); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TTestingNestedMapWithCustomDefaultResetOnLoad + : public TYsonStruct +{ +public: + THashMap<TString, TIntrusivePtr<TSimpleYsonStruct>> NestedMap; + + REGISTER_YSON_STRUCT(TTestingNestedMapWithCustomDefaultResetOnLoad); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("nested_map", &TThis::NestedMap) + .DefaultCtor([] { + return THashMap<TString, TIntrusivePtr<TSimpleYsonStruct>>{ + {"foo", CreateSimpleYsonStruct(42)}, + {"bar", CreateSimpleYsonStruct(7)}, + }; + }) + .ResetOnLoad(); + } +}; + +TEST(TYsonStructTest, NestedMapWithCustomDefaultAndResetOnLoad) +{ + { + auto testInput = TYsonString(TStringBuf("{}")); + auto deserialized = ConvertTo<TIntrusivePtr<TTestingNestedMapWithCustomDefaultResetOnLoad>>(testInput); + + EXPECT_EQ(deserialized->NestedMap.size(), 2u); + EXPECT_EQ(deserialized->NestedMap["foo"]->IntValue, 42); + EXPECT_EQ(deserialized->NestedMap["bar"]->IntValue, 7); + + auto testNode = BuildYsonNodeFluently() + .BeginMap() + .Item("nested_map") + .BeginMap() + .Item("baz") + .BeginMap() + .Item("int_value").Value(33) + .EndMap() + .Item("foo") + .BeginMap() + .Item("int_value").Value(88) + .EndMap() + .EndMap() + .EndMap(); + Deserialize(deserialized, testNode->AsMap()); + EXPECT_EQ(deserialized->NestedMap.size(), 2u); + EXPECT_EQ(deserialized->NestedMap["baz"]->IntValue, 33); + EXPECT_EQ(deserialized->NestedMap["foo"]->IntValue, 88); + } +} + +//////////////////////////////////////////////////////////////////////////////// + } // namespace } // namespace NYT::NYTree diff --git a/yt/yt/core/ytree/yson_struct.cpp b/yt/yt/core/ytree/yson_struct.cpp index 09b98aebf3..0946a1168a 100644 --- a/yt/yt/core/ytree/yson_struct.cpp +++ b/yt/yt/core/ytree/yson_struct.cpp @@ -112,9 +112,9 @@ void TYsonStructBase::SaveParameter(const TString& key, IYsonConsumer* consumer) Meta_->GetParameter(key)->Save(this, consumer); } -void TYsonStructBase::LoadParameter(const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) +void TYsonStructBase::LoadParameter(const TString& key, const NYTree::INodePtr& node) { - Meta_->LoadParameter(this, key, node, mergeStrategy); + Meta_->LoadParameter(this, key, node); } void TYsonStructBase::ResetParameter(const TString& key) diff --git a/yt/yt/core/ytree/yson_struct.h b/yt/yt/core/ytree/yson_struct.h index ea4a488387..fd66023a47 100644 --- a/yt/yt/core/ytree/yson_struct.h +++ b/yt/yt/core/ytree/yson_struct.h @@ -90,7 +90,7 @@ public: // TODO(renadeen): remove this methods. void SaveParameter(const TString& key, NYson::IYsonConsumer* consumer) const; - void LoadParameter(const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy); + void LoadParameter(const TString& key, const NYTree::INodePtr& node); void ResetParameter(const TString& key); std::vector<TString> GetAllParameterAliases(const TString& key) const; diff --git a/yt/yt/core/ytree/yson_struct_detail-inl.h b/yt/yt/core/ytree/yson_struct_detail-inl.h index 6785af863d..0ca9f4ab2f 100644 --- a/yt/yt/core/ytree/yson_struct_detail-inl.h +++ b/yt/yt/core/ytree/yson_struct_detail-inl.h @@ -41,13 +41,8 @@ void LoadFromNode( T& parameter, NYTree::INodePtr node, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> /*recursiveUnrecognizedStrategy*/) { - if (mergeStrategy == EMergeStrategy::Overwrite) { - parameter = T(); - } - try { Deserialize(parameter, node); } catch (const std::exception& ex) { @@ -62,27 +57,12 @@ inline void LoadFromNode( NYTree::INodePtr& parameter, NYTree::INodePtr node, const NYPath::TYPath& /*path*/, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> /*recursiveUnrecognizedStrategy*/) { - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - parameter = node; - break; - } - - case EMergeStrategy::Combine: { - if (!parameter) { - parameter = node; - } else { - parameter = PatchNode(parameter, node); - } - break; - } - - default: - YT_UNIMPLEMENTED(); + if (!parameter) { + parameter = node; + } else { + parameter = PatchNode(parameter, node); } } @@ -92,10 +72,9 @@ void LoadFromNode( TIntrusivePtr<T>& parameterValue, NYTree::INodePtr node, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { - if (!parameterValue || mergeStrategy == EMergeStrategy::Overwrite) { + if (!parameterValue) { parameterValue = New<T>(); } @@ -103,17 +82,7 @@ void LoadFromNode( parameterValue->SetUnrecognizedStrategy(*recursiveUnrecognizedStrategy); } - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: - case EMergeStrategy::Combine: { - parameterValue->Load(node, false, false, path); - break; - } - - default: - YT_UNIMPLEMENTED(); - } + parameterValue->Load(node, false, false, path); } // YsonStructLite or ExternalizedYsonStruct serializer @@ -122,17 +91,8 @@ void LoadFromNode( T& parameter, NYTree::INodePtr node, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> /*recursiveUnrecognizedStrategy*/) { - if (mergeStrategy == EMergeStrategy::Overwrite) { - // NB: We call SetDefaults here instead of plain T() - // because ExternalizedYsonStruct serializer doesn't - // own its data therefore defaulting it would drop the - // reference to the actual object instead of overwriting it. - parameter.SetDefaults(); - } - try { parameter.Load(node, /*postprocess*/ true, /*setDefaults*/ false); } catch (const std::exception& ex) { @@ -147,13 +107,8 @@ void LoadFromNode( T& parameter, NYTree::INodePtr node, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> /*recursiveUnrecognizedStrategy*/) { - if (mergeStrategy == EMergeStrategy::Overwrite) { - parameter = T(); - } - try { DeserializeExternalized(parameter, node, /*postprocess*/ true, /*setDefaults*/ false); } catch (const std::exception& ex) { @@ -168,37 +123,19 @@ void LoadFromNode( std::optional<T>& parameter, NYTree::INodePtr node, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - if (node->GetType() == NYTree::ENodeType::Entity) { - parameter = std::nullopt; - } else { - T value; - LoadFromNode(value, node, path, EMergeStrategy::Overwrite, recursiveUnrecognizedStrategy); - parameter = std::move(value); - } - break; - } - - case EMergeStrategy::Combine: { - if (node->GetType() != NYTree::ENodeType::Entity) { - if (parameter.has_value()) { - LoadFromNode(*parameter, node, path, EMergeStrategy::Combine, recursiveUnrecognizedStrategy); - } else { - T value; - LoadFromNode(value, node, path, EMergeStrategy::Overwrite, recursiveUnrecognizedStrategy); - parameter = std::move(value); - } - } - break; - } + if (node->GetType() == NYTree::ENodeType::Entity) { + parameter = std::nullopt; + return; + } - default: - YT_UNIMPLEMENTED(); + if (parameter.has_value()) { + LoadFromNode(*parameter, node, path, recursiveUnrecognizedStrategy); + } else { + T value; + LoadFromNode(value, node, path, recursiveUnrecognizedStrategy); + parameter = std::move(value); } } @@ -208,28 +145,18 @@ void LoadFromNode( std::vector<T...>& parameter, NYTree::INodePtr node, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - auto listNode = node->AsList(); - auto size = listNode->GetChildCount(); - parameter.resize(size); - for (int i = 0; i < size; ++i) { - LoadFromNode( - parameter[i], - listNode->GetChildOrThrow(i), - path + "/" + NYPath::ToYPathLiteral(i), - EMergeStrategy::Overwrite, - recursiveUnrecognizedStrategy); - } - break; - } - - default: - YT_UNIMPLEMENTED(); + auto listNode = node->AsList(); + auto size = listNode->GetChildCount(); + parameter.clear(); + parameter.reserve(size); + for (int i = 0; i < size; ++i) { + LoadFromNode( + parameter.emplace_back(), + listNode->GetChildOrThrow(i), + path + "/" + NYPath::ToYPathLiteral(i), + recursiveUnrecognizedStrategy); } } @@ -253,43 +180,17 @@ void LoadFromNode( Map<T...>& parameter, NYTree::INodePtr node, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - auto mapNode = node->AsMap(); - parameter.clear(); - for (const auto& [key, child] : mapNode->GetChildren()) { - M value; - LoadFromNode( - value, - child, - path + "/" + NYPath::ToYPathLiteral(key), - EMergeStrategy::Overwrite, - recursiveUnrecognizedStrategy); - parameter.emplace(DeserializeMapKey<typename Map<T...>::key_type>(key), std::move(value)); - } - break; - } - case EMergeStrategy::Combine: { - auto mapNode = node->AsMap(); - for (const auto& [key, child] : mapNode->GetChildren()) { - M value; - LoadFromNode( - value, - child, - path + "/" + NYPath::ToYPathLiteral(key), - EMergeStrategy::Combine, - recursiveUnrecognizedStrategy); - parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value); - } - break; - } - - default: - YT_UNIMPLEMENTED(); + auto mapNode = node->AsMap(); + for (const auto& [key, child] : mapNode->GetChildren()) { + M value; + LoadFromNode( + value, + child, + path + "/" + NYPath::ToYPathLiteral(key), + recursiveUnrecognizedStrategy); + parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value); } } @@ -302,10 +203,9 @@ void LoadFromCursor( T& parameter, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { - LoadFromNode(parameter, NYson::ExtractTo<NYTree::INodePtr>(cursor), path, mergeStrategy, recursiveUnrecognizedStrategy); + LoadFromNode(parameter, NYson::ExtractTo<NYTree::INodePtr>(cursor), path, recursiveUnrecognizedStrategy); } //////////////////////////////////////////////////////////////////////////////// @@ -315,7 +215,6 @@ void LoadFromCursor( TIntrusivePtr<T>& parameterValue, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy); template <class... T> @@ -323,7 +222,6 @@ void LoadFromCursor( std::vector<T...>& parameter, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy); // std::optional @@ -332,7 +230,6 @@ void LoadFromCursor( std::optional<T>& parameter, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy); template <template <typename...> class Map, class... T, class M = typename Map<T...>::mapped_type> @@ -340,7 +237,6 @@ void LoadFromCursor( Map<T...>& parameter, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy); //////////////////////////////////////////////////////////////////////////////// @@ -351,12 +247,11 @@ inline void LoadFromCursor( NYTree::INodePtr& parameter, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { try { auto node = NYson::ExtractTo<INodePtr>(cursor); - LoadFromNode(parameter, std::move(node), path, mergeStrategy, recursiveUnrecognizedStrategy); + LoadFromNode(parameter, std::move(node), path, recursiveUnrecognizedStrategy); } catch (const std::exception& ex) { THROW_ERROR_EXCEPTION("Error loading parameter %v", path) << ex; @@ -369,10 +264,9 @@ void LoadFromCursor( TIntrusivePtr<T>& parameterValue, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { - if (!parameterValue || mergeStrategy == EMergeStrategy::Overwrite) { + if (!parameterValue) { parameterValue = New<T>(); } @@ -380,17 +274,7 @@ void LoadFromCursor( parameterValue->SetUnrecognizedStrategy(*recursiveUnrecognizedStrategy); } - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: - case EMergeStrategy::Combine: { - parameterValue->Load(cursor, false, false, path); - break; - } - - default: - YT_UNIMPLEMENTED(); - } + parameterValue->Load(cursor, /*postprocess*/ false, /*setDefaults*/ false, path); } // std::optional @@ -399,41 +283,20 @@ void LoadFromCursor( std::optional<T>& parameter, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { try { - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - if ((*cursor)->GetType() == NYson::EYsonItemType::EntityValue) { - parameter = std::nullopt; - cursor->Next(); - } else { - T value; - LoadFromCursor(value, cursor, path, EMergeStrategy::Overwrite, recursiveUnrecognizedStrategy); - parameter = std::move(value); - } - break; - } - - case EMergeStrategy::Combine: { - if ((*cursor)->GetType() == NYson::EYsonItemType::EntityValue) { - cursor->Next(); - } else { - if (parameter.has_value()) { - LoadFromCursor(*parameter, cursor, path, EMergeStrategy::Combine, recursiveUnrecognizedStrategy); - } else { - T value; - LoadFromCursor(value, cursor, path, EMergeStrategy::Overwrite, recursiveUnrecognizedStrategy); - parameter = std::move(value); - } - } - break; + if ((*cursor)->GetType() == NYson::EYsonItemType::EntityValue) { + parameter = std::nullopt; + cursor->Next(); + } else { + if (parameter.has_value()) { + LoadFromCursor(*parameter, cursor, path, recursiveUnrecognizedStrategy); + } else { + T value; + LoadFromCursor(value, cursor, path, recursiveUnrecognizedStrategy); + parameter = std::move(value); } - - default: - YT_UNIMPLEMENTED(); } } catch (const std::exception& ex) { THROW_ERROR_EXCEPTION("Error loading parameter %v", path) @@ -447,30 +310,19 @@ void LoadFromCursor( std::vector<T...>& parameter, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { try { - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - parameter.clear(); - int index = 0; - cursor->ParseList([&](NYson::TYsonPullParserCursor* cursor) { - LoadFromCursor( - parameter.emplace_back(), - cursor, - path + "/" + NYPath::ToYPathLiteral(index), - EMergeStrategy::Overwrite, - recursiveUnrecognizedStrategy); - ++index; - }); - break; - } - - default: - YT_UNIMPLEMENTED(); - } + parameter.clear(); + int index = 0; + cursor->ParseList([&](NYson::TYsonPullParserCursor* cursor) { + LoadFromCursor( + parameter.emplace_back(), + cursor, + path + "/" + NYPath::ToYPathLiteral(index), + recursiveUnrecognizedStrategy); + ++index; + }); } catch (const std::exception& ex) { THROW_ERROR_EXCEPTION("Error loading parameter %v", path) << ex; @@ -483,44 +335,19 @@ void LoadFromCursor( Map<T...>& parameter, NYson::TYsonPullParserCursor* cursor, const NYPath::TYPath& path, - EMergeStrategy mergeStrategy, std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy) { try { - auto doParse = [&] (const auto& setterOrEmplacer, EMergeStrategy mergeStrategy) { - cursor->ParseMap([&] (NYson::TYsonPullParserCursor* cursor) { - auto key = ExtractTo<TString>(cursor); - M value; - LoadFromCursor( - value, - cursor, - path + "/" + NYPath::ToYPathLiteral(key), - mergeStrategy, - recursiveUnrecognizedStrategy); - setterOrEmplacer(key, std::move(value)); - }); - }; - - switch (mergeStrategy) { - case EMergeStrategy::Default: - case EMergeStrategy::Overwrite: { - parameter.clear(); - auto emplacer = [&] (auto key, M&& value) { - parameter.emplace(DeserializeMapKey<typename Map<T...>::key_type>(key), std::move(value)); - }; - doParse(emplacer, EMergeStrategy::Overwrite); - break; - } - case EMergeStrategy::Combine: { - auto setter = [&] (auto key, M&& value) { - parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value); - }; - doParse(setter, EMergeStrategy::Combine); - break; - } - default: - YT_UNIMPLEMENTED(); - } + cursor->ParseMap([&] (NYson::TYsonPullParserCursor* cursor) { + auto key = ExtractTo<TString>(cursor); + M value; + LoadFromCursor( + value, + cursor, + path + "/" + NYPath::ToYPathLiteral(key), + recursiveUnrecognizedStrategy); + parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value); + }); } catch (const std::exception& ex) { THROW_ERROR_EXCEPTION("Error loading parameter %v", path) << ex; @@ -645,7 +472,58 @@ inline void InvokeForComposites(const Map<T...>* parameter, const F& func) //////////////////////////////////////////////////////////////////////////////// -} // namespace NYsonStructDetail +// all +template <class T> +inline void ResetOnLoad(T& parameter) +{ + parameter = T(); +} + +// TYsonStruct +template <std::derived_from<TYsonStruct> T> +inline void ResetOnLoad(TIntrusivePtr<T>& parameter) +{ + parameter = New<T>(); +} + +// TYsonStructLite or TExternalizedYsonStruct Serializer +template <std::derived_from<TYsonStructLite> T> +inline void ResetOnLoad(T& parameter) +{ + parameter.SetDefaults(); +} + +// INodePtr +template <> +inline void ResetOnLoad(INodePtr& parameter) +{ + parameter.Reset(); +} + +// std::optional +template <class T> +inline void ResetOnLoad(std::optional<T>& parameter) +{ + parameter.reset(); +} + +// std::vector +template <class T> +inline void ResetOnLoad(std::vector<T>& parameter) +{ + parameter.clear(); +} + +// any map +template <template <typename...> class Map, class... T, class M = typename Map<T...>::mapped_type> +inline void ResetOnLoad(Map<T...>& parameter) +{ + parameter.clear(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NPrivate //////////////////////////////////////////////////////////////////////////////// @@ -679,7 +557,6 @@ template <class TValue> TYsonStructParameter<TValue>::TYsonStructParameter(TString key, std::unique_ptr<IYsonFieldAccessor<TValue>> fieldAccessor) : Key_(std::move(key)) , FieldAccessor_(std::move(fieldAccessor)) - , MergeStrategy_(EMergeStrategy::Default) { } template <class TValue> @@ -689,11 +566,13 @@ void TYsonStructParameter<TValue>::Load( const TLoadParameterOptions& options) { if (node) { + if (ResetOnLoad_) { + NPrivate::ResetOnLoad(FieldAccessor_->GetValue(self)); + } NPrivate::LoadFromNode( FieldAccessor_->GetValue(self), std::move(node), options.Path, - options.MergeStrategy.value_or(MergeStrategy_), options.RecursiveUnrecognizedRecursively); } else if (!Optional_) { THROW_ERROR_EXCEPTION("Missing required parameter %v", @@ -702,41 +581,19 @@ void TYsonStructParameter<TValue>::Load( } template <class TValue> -void TYsonStructParameter<TValue>::SafeLoad( - TYsonStructBase* self, - NYTree::INodePtr node, - const TLoadParameterOptions& options, - const std::function<void()>& validate) -{ - if (node) { - TValue oldValue = FieldAccessor_->GetValue(self); - try { - NPrivate::LoadFromNode( - FieldAccessor_->GetValue(self), - node, - options.Path, - options.MergeStrategy.value_or(MergeStrategy_), - /*recursivelyUnrecognizedStrategy*/ std::nullopt); - validate(); - } catch (const std::exception ex) { - FieldAccessor_->GetValue(self) = oldValue; - throw; - } - } -} - -template <class TValue> void TYsonStructParameter<TValue>::Load( TYsonStructBase* self, NYson::TYsonPullParserCursor* cursor, const TLoadParameterOptions& options) { if (cursor) { + if (ResetOnLoad_) { + NPrivate::ResetOnLoad(FieldAccessor_->GetValue(self)); + } NPrivate::LoadFromCursor( FieldAccessor_->GetValue(self), cursor, options.Path, - options.MergeStrategy.value_or(MergeStrategy_), options.RecursiveUnrecognizedRecursively); } else if (!Optional_) { THROW_ERROR_EXCEPTION("Missing required parameter %v", @@ -747,18 +604,18 @@ void TYsonStructParameter<TValue>::Load( template <class TValue> void TYsonStructParameter<TValue>::SafeLoad( TYsonStructBase* self, - NYson::TYsonPullParserCursor* cursor, + NYTree::INodePtr node, const TLoadParameterOptions& options, const std::function<void()>& validate) { - if (cursor) { + if (node) { TValue oldValue = FieldAccessor_->GetValue(self); try { - NPrivate::LoadFromCursor( + FieldAccessor_->GetValue(self) = TValue(); + NPrivate::LoadFromNode( FieldAccessor_->GetValue(self), - cursor, + node, options.Path, - options.MergeStrategy.value_or(MergeStrategy_), /*recursivelyUnrecognizedStrategy*/ std::nullopt); validate(); } catch (const std::exception ex) { @@ -839,6 +696,13 @@ TYsonStructParameter<TValue>& TYsonStructParameter<TValue>::Alias(const TString& } template <class TValue> +TYsonStructParameter<TValue>& TYsonStructParameter<TValue>::ResetOnLoad() +{ + ResetOnLoad_ = true; + return *this; +} + +template <class TValue> const std::vector<TString>& TYsonStructParameter<TValue>::GetAliases() const { return Aliases_; @@ -921,13 +785,6 @@ TYsonStructParameter<TValue>& TYsonStructParameter<TValue>::CheckThat(TPostproce } template <class TValue> -TYsonStructParameter<TValue>& TYsonStructParameter<TValue>::MergeBy(EMergeStrategy strategy) -{ - MergeStrategy_ = strategy; - return *this; -} - -template <class TValue> IMapNodePtr TYsonStructParameter<TValue>::GetRecursiveUnrecognized(const TYsonStructBase* self) const { return NPrivate::TGetRecursiveUnrecognized<TValue>::Do(FieldAccessor_->GetValue(self)); diff --git a/yt/yt/core/ytree/yson_struct_detail.cpp b/yt/yt/core/ytree/yson_struct_detail.cpp index 48919a5d22..5ccb78dcbc 100644 --- a/yt/yt/core/ytree/yson_struct_detail.cpp +++ b/yt/yt/core/ytree/yson_struct_detail.cpp @@ -68,7 +68,7 @@ IYsonStructParameterPtr TYsonStructMeta::GetParameter(const TString& keyOrAlias) THROW_ERROR_EXCEPTION("Key or alias %Qv not found in yson struct", keyOrAlias); } -void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) const +void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node) const { const auto& parameter = GetParameter(key); auto validate = [&] () { @@ -87,7 +87,6 @@ void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const TString& key, }; auto loadOptions = TLoadParameterOptions{ .Path = "", - .MergeStrategy = mergeStrategy }; parameter->SafeLoad(target, node, loadOptions, validate); diff --git a/yt/yt/core/ytree/yson_struct_detail.h b/yt/yt/core/ytree/yson_struct_detail.h index e6d5a1b595..046a66188d 100644 --- a/yt/yt/core/ytree/yson_struct_detail.h +++ b/yt/yt/core/ytree/yson_struct_detail.h @@ -18,7 +18,6 @@ struct TLoadParameterOptions { NYPath::TYPath Path; std::optional<EUnrecognizedStrategy> RecursiveUnrecognizedRecursively; - std::optional<EMergeStrategy> MergeStrategy; }; //////////////////////////////////////////////////////////////////////////////// @@ -42,12 +41,6 @@ struct IYsonStructParameter const TLoadParameterOptions& options, const std::function<void()>& validate) = 0; - virtual void SafeLoad( - TYsonStructBase* self, - NYson::TYsonPullParserCursor* cursor, - const TLoadParameterOptions& options, - const std::function<void()>& validate) = 0; - virtual void Save(const TYsonStructBase* self, NYson::IYsonConsumer* consumer) const = 0; virtual void Postprocess(const TYsonStructBase* self, const NYPath::TYPath& path) const = 0; @@ -77,7 +70,7 @@ struct IYsonStructMeta virtual const THashSet<TString>& GetRegisteredKeys() const = 0; virtual void Postprocess(TYsonStructBase* target, const TYPath& path) const = 0; virtual IYsonStructParameterPtr GetParameter(const TString& keyOrAlias) const = 0; - virtual void LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) const = 0; + virtual void LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node) const = 0; virtual void LoadStruct( TYsonStructBase* target, @@ -118,7 +111,7 @@ public: const THashSet<TString>& GetRegisteredKeys() const override; IYsonStructParameterPtr GetParameter(const TString& keyOrAlias) const override; - void LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) const override; + void LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node) const override; void Postprocess(TYsonStructBase* target, const TYPath& path) const override; @@ -223,12 +216,6 @@ public: NYTree::INodePtr node, const TLoadParameterOptions& options) override; - void SafeLoad( - TYsonStructBase* self, - NYTree::INodePtr node, - const TLoadParameterOptions& options, - const std::function<void()>& validate) override; - void Load( TYsonStructBase* self, NYson::TYsonPullParserCursor* cursor, @@ -236,7 +223,7 @@ public: void SafeLoad( TYsonStructBase* self, - NYson::TYsonPullParserCursor* cursor, + NYTree::INodePtr node, const TLoadParameterOptions& options, const std::function<void()>& validate) override; @@ -278,8 +265,8 @@ public: TYsonStructParameter& NonEmpty(); // Register alias for parameter. Used in deserialization. TYsonStructParameter& Alias(const TString& name); - // Set merge strategy for parameter - TYsonStructParameter& MergeBy(EMergeStrategy strategy); + // Set field to T() (or suitable analogue) before deserializations. + TYsonStructParameter& ResetOnLoad(); // Register constructor with parameters as initializer of default value for ref-counted class. template <class... TArgs> @@ -293,9 +280,9 @@ private: bool SerializeDefault_ = true; std::vector<TPostprocessor> Postprocessors_; std::vector<TString> Aliases_; - EMergeStrategy MergeStrategy_ = EMergeStrategy::Default; bool TriviallyInitializedIntrusivePtr_ = false; bool Optional_ = false; + bool ResetOnLoad_ = false; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/yson_struct_enum.h b/yt/yt/core/ytree/yson_struct_enum.h index 47824666fc..a35fba6d0d 100644 --- a/yt/yt/core/ytree/yson_struct_enum.h +++ b/yt/yt/core/ytree/yson_struct_enum.h @@ -6,13 +6,7 @@ namespace NYT::NYTree { -//////////////////////////////////////////////////////////////////////////////// - -DEFINE_ENUM(EMergeStrategy, - (Default) - (Overwrite) - (Combine) -); +/////////////////////////////////////////////////////////////////////////////// DEFINE_ENUM(EUnrecognizedStrategy, (Drop) diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index f20d71c352..f8977a71a4 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -271,6 +271,38 @@ enum EJobSpecSource JSS_AUTO = 0xFFFF; } +enum EQueryEngine +{ + QE_UNKNOWN = 100; + + QE_QL = 0; + QE_YQL = 1; + QE_CHYT = 2; + QE_MOCK = 3; + QE_SPYT = 4; +} + +enum EContentType +{ + CT_RAW_INLINE_DATA = 0; + CT_URL = 1; +} + +enum EQueryState +{ + QS_UNKNOWN = 100; + + QS_DRAFT = 0; + QS_PENDING = 1; + QS_RUNNING = 2; + QS_ABORTING = 3; + QS_ABORTED = 4; + QS_COMPLETING = 5; + QS_COMPLETED = 6; + QS_FAILING = 7; + QS_FAILED = 8; +} + // COMPAT(max42). // A legacy analog of NYT.NYTree.NProto.TAttributeFilter. // It is different in that universal filter is encoded via all = true @@ -3004,3 +3036,174 @@ message TCheckPermissionByAclResult optional string subject_name = 3; repeated string missing_subjects = 4; } + +//////////////////////////////////////////////////////////////////////////////// +// Query Tracker +//////////////////////////////////////////////////////////////////////////////// + +message TReqStartQuery +{ + message TQueryFile + { + required string name = 1; + required string content = 2; + required EContentType type = 3; + } + + required string query_tracker_stage = 1; + required EQueryEngine engine = 2; + required string query = 3; + optional bytes settings = 4; // YSON + optional bool draft = 5; + optional bytes annotations = 6; // YSON + repeated TQueryFile files = 7; + optional string access_control_object = 8; +} + +message TRspStartQuery +{ + optional NYT.NProto.TGuid query_id = 1; +} + +//////////////////////////////////////////////////////////////////////////////// + +message TReqAbortQuery +{ + required string query_tracker_stage = 1; + required NYT.NProto.TGuid query_id = 2; + optional string abort_message = 3; +} + +message TRspAbortQuery +{ +} + +//////////////////////////////////////////////////////////////////////////////// + +message TReqGetQueryResult +{ + required string query_tracker_stage = 1; + required NYT.NProto.TGuid query_id = 2; + required int64 result_index = 3; +} + +message TRspGetQueryResult +{ + required NYT.NProto.TGuid query_id = 1; + required int64 result_index = 2; + optional NYT.NProto.TError error = 3; + optional TTableSchema schema = 4; + required NYT.NChunkClient.NProto.TDataStatistics data_statistics = 5; + required bool is_truncated = 6; +} + +//////////////////////////////////////////////////////////////////////////////// + +message TReqReadQueryResult +{ + message TColumns + { + repeated string items = 1; + } + + required string query_tracker_stage = 1; + required NYT.NProto.TGuid query_id = 2; + required int64 result_index = 3; + optional TColumns columns = 4; + optional int64 lower_row_index = 5; + optional int64 upper_row_index = 6; +} + +message TRspReadQueryResult +{ + required TRowsetDescriptor rowset_descriptor = 1; +} + +//////////////////////////////////////////////////////////////////////////////// + +message TQuery +{ + required NYT.NProto.TGuid id = 1; + optional EQueryEngine engine = 2; + optional string query = 3; + optional bytes files = 4; // YSON + optional uint64 start_time = 5; // TInstant + optional uint64 finish_time = 6; // TInstant + optional bytes settings = 7; // YSON + optional string user = 8; + optional string access_control_object = 9; + optional EQueryState state = 10; + optional int64 result_count = 11; + optional bytes progress = 12; // YSON + optional NYT.NProto.TError error = 13; + optional bytes annotations = 14; // YSON + optional NYT.NYTree.NProto.TAttributeDictionary other_attributes = 15; +} + +//////////////////////////////////////////////////////////////////////////////// + +message TReqGetQuery +{ + required string query_tracker_stage = 1; + required NYT.NProto.TGuid query_id = 2; + optional NYT.NYTree.NProto.TAttributeFilter attributes = 3; + optional uint64 timestamp = 4; +} + +message TRspGetQuery +{ + optional TQuery query = 1; +} + +//////////////////////////////////////////////////////////////////////////////// + +message TReqListQueries +{ + required string query_tracker_stage = 1; + optional uint64 from_time = 2; // TInstant + optional uint64 to_time = 3; // TInstant + optional uint64 cursor_time = 4; // TInstant + optional EOperationSortDirection cursor_direction = 5 [default = OSD_PAST]; + optional string user_filter = 6; + optional EQueryState state_filter = 7; + optional EQueryEngine engine_filter = 8; + optional string substr_filter = 9; + optional uint64 limit = 10 [default = 100]; + optional NYT.NYTree.NProto.TAttributeFilter attributes = 11; +} + +message TRspListQueries +{ + repeated TQuery queries = 1; + required bool incomplete = 2; + required uint64 timestamp = 3; +} + +//////////////////////////////////////////////////////////////////////////////// + +message TReqAlterQuery +{ + required string query_tracker_stage = 1; + required NYT.NProto.TGuid query_id = 2; + optional bytes annotations = 3; // YSON + optional string access_control_object = 4; +} + +message TRspAlterQuery +{ +} + +//////////////////////////////////////////////////////////////////////////////// + +message TReqGetQueryTrackerInfo +{ + required string query_tracker_stage = 1; + optional NYT.NYTree.NProto.TAttributeFilter attributes = 2; +} + +message TRspGetQueryTrackerInfo +{ + required string cluster_name = 1; + required bytes supported_features = 2; // YSON + repeated string access_control_objects = 3; +} |