aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorrobot-piglet <robot-piglet@yandex-team.com>2024-03-01 17:42:44 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2024-03-01 17:53:54 +0300
commit280ea5e89bc75e2f23148f30347dbd2795d160a5 (patch)
tree959129f5896a995afd002865673f73bde9fecff6
parentc90958a82ed1e508a3db6b0e75e682ae8f3f5659 (diff)
downloadydb-280ea5e89bc75e2f23148f30347dbd2795d160a5.tar.gz
Intermediate changes
-rw-r--r--yt/yt/client/api/query_tracker_client.h4
-rw-r--r--yt/yt/client/api/rpc_proxy/api_service_proxy.h10
-rw-r--r--yt/yt/client/api/rpc_proxy/client_impl.cpp227
-rw-r--r--yt/yt/client/api/rpc_proxy/helpers.cpp222
-rw-r--r--yt/yt/client/api/rpc_proxy/helpers.h19
-rw-r--r--yt/yt/client/table_client/config.cpp3
-rw-r--r--yt/yt/core/ytree/unittests/yson_struct_ut.cpp166
-rw-r--r--yt/yt/core/ytree/yson_struct.cpp4
-rw-r--r--yt/yt/core/ytree/yson_struct.h2
-rw-r--r--yt/yt/core/ytree/yson_struct_detail-inl.h421
-rw-r--r--yt/yt/core/ytree/yson_struct_detail.cpp3
-rw-r--r--yt/yt/core/ytree/yson_struct_detail.h25
-rw-r--r--yt/yt/core/ytree/yson_struct_enum.h8
-rw-r--r--yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto203
14 files changed, 962 insertions, 355 deletions
diff --git a/yt/yt/client/api/query_tracker_client.h b/yt/yt/client/api/query_tracker_client.h
index f85a6757d2..e2cb81d27e 100644
--- a/yt/yt/client/api/query_tracker_client.h
+++ b/yt/yt/client/api/query_tracker_client.h
@@ -15,7 +15,7 @@ struct TQueryTrackerOptions
TString QueryTrackerStage = "production";
};
-DEFINE_ENUM(ContentType,
+DEFINE_ENUM(EContentType,
((RawInlineData) (0))
((Url) (1))
);
@@ -25,7 +25,7 @@ struct TQueryFile
{
TString Name;
TString Content;
- ContentType Type;
+ EContentType Type;
REGISTER_YSON_STRUCT(TQueryFile);
diff --git a/yt/yt/client/api/rpc_proxy/api_service_proxy.h b/yt/yt/client/api/rpc_proxy/api_service_proxy.h
index 6aaf2c1e36..d90e1ede0a 100644
--- a/yt/yt/client/api/rpc_proxy/api_service_proxy.h
+++ b/yt/yt/client/api/rpc_proxy/api_service_proxy.h
@@ -183,6 +183,16 @@ public:
DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, PausePipeline);
DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, GetPipelineStatus);
+ // Query tracker
+ DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, StartQuery);
+ DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, AbortQuery);
+ DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, GetQueryResult);
+ DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, ReadQueryResult);
+ DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, GetQuery);
+ DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, ListQueries);
+ DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, AlterQuery);
+ DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, GetQueryTrackerInfo);
+
// Misc
DEFINE_RPC_PROXY_METHOD(NRpcProxy::NProto, CheckClusterLiveness);
};
diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp
index 1089d99179..6daa563030 100644
--- a/yt/yt/client/api/rpc_proxy/client_impl.cpp
+++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp
@@ -2019,60 +2019,237 @@ TFuture<TRequestRestartResult> TClient::RequestRestart(
}
TFuture<NQueryTrackerClient::TQueryId> TClient::StartQuery(
- NQueryTrackerClient::EQueryEngine /*engine*/,
- const TString& /*query*/,
- const TStartQueryOptions& /*options*/)
+ NQueryTrackerClient::EQueryEngine engine,
+ const TString& query,
+ const TStartQueryOptions& options)
{
- ThrowUnimplemented("StartQuery");
+ auto proxy = CreateApiServiceProxy();
+
+ auto req = proxy.StartQuery();
+ SetTimeoutOptions(*req, options);
+
+ req->set_query_tracker_stage(options.QueryTrackerStage);
+ req->set_engine(NProto::ConvertQueryEngineToProto(engine));
+ req->set_query(query);
+ req->set_draft(options.Draft);
+
+ if (options.Settings) {
+ req->set_settings(ConvertToYsonString(options.Settings).ToString());
+ }
+ if (options.Annotations) {
+ req->set_annotations(ConvertToYsonString(options.Annotations).ToString());
+ }
+ if (options.AccessControlObject) {
+ req->set_access_control_object(*options.AccessControlObject);
+ }
+
+ for (const auto& file : options.Files) {
+ auto* protoFile = req->add_files();
+ protoFile->set_name(file->Name);
+ protoFile->set_content(file->Content);
+ protoFile->set_type(static_cast<NProto::EContentType>(file->Type));
+ }
+
+ return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspStartQueryPtr& rsp) {
+ return FromProto<NQueryTrackerClient::TQueryId>(rsp->query_id());
+ }));
}
TFuture<void> TClient::AbortQuery(
- NQueryTrackerClient::TQueryId /*queryId*/,
- const TAbortQueryOptions& /*options*/)
+ NQueryTrackerClient::TQueryId queryId,
+ const TAbortQueryOptions& options)
{
- ThrowUnimplemented("AbortQuery");
+ auto proxy = CreateApiServiceProxy();
+
+ auto req = proxy.AbortQuery();
+ SetTimeoutOptions(*req, options);
+
+ req->set_query_tracker_stage(options.QueryTrackerStage);
+ ToProto(req->mutable_query_id(), queryId);
+
+ if (options.AbortMessage) {
+ req->set_abort_message(*options.AbortMessage);
+ }
+
+ return req->Invoke().AsVoid();
}
TFuture<TQueryResult> TClient::GetQueryResult(
- NQueryTrackerClient::TQueryId /*queryId*/,
- i64 /*resultIndex*/,
- const TGetQueryResultOptions& /*options*/)
+ NQueryTrackerClient::TQueryId queryId,
+ i64 resultIndex,
+ const TGetQueryResultOptions& options)
{
- ThrowUnimplemented("GetQueryResult");
+ auto proxy = CreateApiServiceProxy();
+
+ auto req = proxy.GetQueryResult();
+ SetTimeoutOptions(*req, options);
+
+ req->set_query_tracker_stage(options.QueryTrackerStage);
+ ToProto(req->mutable_query_id(), queryId);
+ req->set_result_index(resultIndex);
+
+ return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetQueryResultPtr& rsp) {
+ return TQueryResult{
+ .Id = FromProto<NQueryTrackerClient::TQueryId>(rsp->query_id()),
+ .ResultIndex = rsp->result_index(),
+ .Error = FromProto<TError>(rsp->error()),
+ .Schema = rsp->has_schema() ? FromProto<NTableClient::TTableSchemaPtr>(rsp->schema()) : nullptr,
+ .DataStatistics = FromProto<NChunkClient::NProto::TDataStatistics>(rsp->data_statistics()),
+ .IsTruncated = rsp->is_truncated(),
+ };
+ }));
}
TFuture<IUnversionedRowsetPtr> TClient::ReadQueryResult(
- NQueryTrackerClient::TQueryId /*queryId*/,
- i64 /*resultIndex*/,
- const TReadQueryResultOptions& /*options*/)
+ NQueryTrackerClient::TQueryId queryId,
+ i64 resultIndex,
+ const TReadQueryResultOptions& options)
{
- ThrowUnimplemented("AbortQuery");
+ auto proxy = CreateApiServiceProxy();
+
+ auto req = proxy.ReadQueryResult();
+ SetTimeoutOptions(*req, options);
+
+ req->set_query_tracker_stage(options.QueryTrackerStage);
+ ToProto(req->mutable_query_id(), queryId);
+ req->set_result_index(resultIndex);
+
+ if (options.Columns) {
+ auto* protoColumns = req->mutable_columns();
+ for (const auto& column : *options.Columns) {
+ protoColumns->add_items(column);
+ }
+ }
+ if (options.LowerRowIndex) {
+ req->set_lower_row_index(*options.LowerRowIndex);
+ }
+ if (options.UpperRowIndex) {
+ req->set_upper_row_index(*options.UpperRowIndex);
+ }
+
+ return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspReadQueryResultPtr& rsp) {
+ return DeserializeRowset<TUnversionedRow>(
+ rsp->rowset_descriptor(),
+ MergeRefsToRef<TRpcProxyClientBufferTag>(rsp->Attachments()));
+ }));
}
TFuture<TQuery> TClient::GetQuery(
- NQueryTrackerClient::TQueryId /*queryId*/,
- const TGetQueryOptions& /*options*/)
+ NQueryTrackerClient::TQueryId queryId,
+ const TGetQueryOptions& options)
{
- ThrowUnimplemented("GetQuery");
+ auto proxy = CreateApiServiceProxy();
+
+ auto req = proxy.GetQuery();
+ SetTimeoutOptions(*req, options);
+
+ req->set_query_tracker_stage(options.QueryTrackerStage);
+ ToProto(req->mutable_query_id(), queryId);
+
+ if (options.Attributes) {
+ ToProto(req->mutable_attributes(), options.Attributes);
+ }
+ if (options.Timestamp) {
+ req->set_timestamp(ToProto<i64>(options.Timestamp));
+ }
+
+ return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetQueryPtr& rsp) {
+ return FromProto<TQuery>(rsp->query());
+ }));
}
TFuture<TListQueriesResult> TClient::ListQueries(
- const TListQueriesOptions& /*options*/)
+ const TListQueriesOptions& options)
{
- ThrowUnimplemented("ListQueries");
+ auto proxy = CreateApiServiceProxy();
+
+ auto req = proxy.ListQueries();
+ SetTimeoutOptions(*req, options);
+
+ req->set_query_tracker_stage(options.QueryTrackerStage);
+
+ if (options.FromTime) {
+ req->set_from_time(NYT::ToProto<i64>(*options.FromTime));
+ }
+ if (options.ToTime) {
+ req->set_to_time(NYT::ToProto<i64>(*options.ToTime));
+ }
+ if (options.CursorTime) {
+ req->set_cursor_time(NYT::ToProto<i64>(*options.CursorTime));
+ }
+ req->set_cursor_direction(static_cast<NProto::EOperationSortDirection>(options.CursorDirection));
+
+ if (options.UserFilter) {
+ req->set_user_filter(*options.UserFilter);
+ }
+ if (options.StateFilter) {
+ req->set_state_filter(NProto::ConvertQueryStateToProto(*options.StateFilter));
+ }
+ if (options.EngineFilter) {
+ req->set_engine_filter(NProto::ConvertQueryEngineToProto(*options.EngineFilter));
+ }
+ if (options.SubstrFilter) {
+ req->set_substr_filter(*options.SubstrFilter);
+ }
+
+ req->set_limit(options.Limit);
+
+ if (options.Attributes) {
+ ToProto(req->mutable_attributes(), options.Attributes);
+ }
+
+ return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspListQueriesPtr& rsp) {
+ return TListQueriesResult{
+ .Queries = FromProto<std::vector<TQuery>>(rsp->queries()),
+ .Incomplete = rsp->incomplete(),
+ .Timestamp = rsp->timestamp(),
+ };
+ }));
}
TFuture<void> TClient::AlterQuery(
- NQueryTrackerClient::TQueryId /*queryId*/,
- const TAlterQueryOptions& /*options*/)
+ NQueryTrackerClient::TQueryId queryId,
+ const TAlterQueryOptions& options)
{
- ThrowUnimplemented("AlterQuery");
+ auto proxy = CreateApiServiceProxy();
+
+ auto req = proxy.AlterQuery();
+ SetTimeoutOptions(*req, options);
+
+ req->set_query_tracker_stage(options.QueryTrackerStage);
+ ToProto(req->mutable_query_id(), queryId);
+
+ if (options.Annotations) {
+ req->set_annotations(ConvertToYsonString(options.Annotations).ToString());
+ }
+ if (options.AccessControlObject) {
+ req->set_access_control_object(*options.AccessControlObject);
+ }
+
+ return req->Invoke().AsVoid();
}
TFuture<TGetQueryTrackerInfoResult> TClient::GetQueryTrackerInfo(
- const TGetQueryTrackerInfoOptions& /*options*/)
+ const TGetQueryTrackerInfoOptions& options)
{
- ThrowUnimplemented("GetQueryTrackerInfo");
+ auto proxy = CreateApiServiceProxy();
+
+ auto req = proxy.GetQueryTrackerInfo();
+ SetTimeoutOptions(*req, options);
+
+ req->set_query_tracker_stage(options.QueryTrackerStage);
+
+ if (options.Attributes) {
+ ToProto(req->mutable_attributes(), options.Attributes);
+ }
+
+ return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetQueryTrackerInfoPtr& rsp) {
+ return TGetQueryTrackerInfoResult{
+ .ClusterName = rsp->cluster_name(),
+ .SupportedFeatures = TYsonString(rsp->supported_features()),
+ .AccessControlObjects = FromProto<std::vector<TString>>(rsp->access_control_objects()),
+ };
+ }));
}
TFuture<NBundleControllerClient::TBundleConfigDescriptorPtr> TClient::GetBundleConfig(
diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp
index fdbf14614a..1dff960561 100644
--- a/yt/yt/client/api/rpc_proxy/helpers.cpp
+++ b/yt/yt/client/api/rpc_proxy/helpers.cpp
@@ -1332,6 +1332,136 @@ void FromProto(
FromProto(&manifest->Clusters, protoManifest.clusters());
}
+void ToProto(
+ NProto::TQuery* protoQuery,
+ const NApi::TQuery& query)
+{
+ protoQuery->Clear();
+
+ ToProto(protoQuery->mutable_id(), query.Id);
+
+ if (query.Engine) {
+ protoQuery->set_engine(ConvertQueryEngineToProto(*query.Engine));
+ }
+ if (query.Query) {
+ protoQuery->set_query(*query.Query);
+ }
+ if (query.Files) {
+ protoQuery->set_files(query.Files->ToString());
+ }
+ if (query.StartTime) {
+ protoQuery->set_start_time(NYT::ToProto<i64>(*query.StartTime));
+ }
+ if (query.FinishTime) {
+ protoQuery->set_start_time(NYT::ToProto<i64>(*query.FinishTime));
+ }
+ if (query.Settings) {
+ protoQuery->set_settings(query.Settings.ToString());
+ }
+ if (query.User) {
+ protoQuery->set_user(*query.User);
+ }
+ if (query.AccessControlObject) {
+ protoQuery->set_access_control_object(*query.AccessControlObject);
+ }
+ if (query.State) {
+ protoQuery->set_state(ConvertQueryStateToProto(*query.State));
+ }
+ if (query.ResultCount) {
+ protoQuery->set_result_count(*query.ResultCount);
+ }
+ if (query.Progress) {
+ protoQuery->set_progress(query.Progress.ToString());
+ }
+ if (query.Error) {
+ ToProto(protoQuery->mutable_error(), *query.Error);
+ }
+ if (query.Annotations) {
+ protoQuery->set_annotations(query.Annotations.ToString());
+ }
+ if (query.OtherAttributes) {
+ ToProto(protoQuery->mutable_other_attributes(), *query.OtherAttributes);
+ }
+}
+
+void FromProto(
+ NApi::TQuery* query,
+ const NProto::TQuery& protoQuery)
+{
+ FromProto(&query->Id, protoQuery.id());
+
+ if (protoQuery.has_engine()) {
+ query->Engine = ConvertQueryEngineFromProto(protoQuery.engine());
+ } else {
+ query->Engine.reset();
+ }
+ if (protoQuery.has_query()) {
+ query->Query = protoQuery.query();
+ } else {
+ query->Query.reset();
+ }
+ if (protoQuery.has_files()) {
+ query->Files = TYsonString(protoQuery.files());
+ } else {
+ query->Files.reset();
+ }
+ if (protoQuery.has_start_time()) {
+ query->StartTime = TInstant::FromValue(protoQuery.start_time());
+ } else {
+ query->StartTime.reset();
+ }
+ if (protoQuery.has_finish_time()) {
+ query->FinishTime = TInstant::FromValue(protoQuery.finish_time());
+ } else {
+ query->FinishTime.reset();
+ }
+ if (protoQuery.has_settings()) {
+ query->Settings = TYsonString(protoQuery.settings());
+ } else {
+ query->Settings = TYsonString{};
+ }
+ if (protoQuery.has_user()) {
+ query->User = protoQuery.user();
+ } else {
+ query->User.reset();
+ }
+ if (protoQuery.has_access_control_object()) {
+ query->AccessControlObject = protoQuery.access_control_object();
+ } else {
+ query->AccessControlObject.reset();
+ }
+ if (protoQuery.has_state()) {
+ query->State = ConvertQueryStateFromProto(protoQuery.state());
+ } else {
+ query->State.reset();
+ }
+ if (protoQuery.result_count()) {
+ query->ResultCount = protoQuery.result_count();
+ } else {
+ query->ResultCount.reset();
+ }
+ if (protoQuery.has_progress()) {
+ query->Progress = TYsonString(protoQuery.progress());
+ } else {
+ query->Progress = TYsonString{};
+ }
+ if (protoQuery.has_error()) {
+ query->Error = FromProto<TError>(protoQuery.error());
+ } else {
+ query->Error.reset();
+ }
+ if (protoQuery.has_annotations()) {
+ query->Annotations = TYsonString(protoQuery.annotations());
+ } else {
+ query->Annotations = TYsonString{};
+ }
+ if (protoQuery.has_other_attributes()) {
+ query->OtherAttributes = NYTree::FromProto(protoQuery.other_attributes());
+ } else if (query->OtherAttributes) {
+ query->OtherAttributes->Clear();
+ }
+}
+
////////////////////////////////////////////////////////////////////////////////
// ENUMS
////////////////////////////////////////////////////////////////////////////////
@@ -1648,6 +1778,98 @@ NJobTrackerClient::EJobState ConvertJobStateFromProto(
YT_ABORT();
}
+NProto::EQueryEngine ConvertQueryEngineToProto(
+ NQueryTrackerClient::EQueryEngine queryEngine)
+{
+ switch (queryEngine) {
+ case NQueryTrackerClient::EQueryEngine::Ql:
+ return NProto::EQueryEngine::QE_QL;
+ case NQueryTrackerClient::EQueryEngine::Yql:
+ return NProto::EQueryEngine::QE_YQL;
+ case NQueryTrackerClient::EQueryEngine::Chyt:
+ return NProto::EQueryEngine::QE_CHYT;
+ case NQueryTrackerClient::EQueryEngine::Mock:
+ return NProto::EQueryEngine::QE_MOCK;
+ case NQueryTrackerClient::EQueryEngine::Spyt:
+ return NProto::EQueryEngine::QE_SPYT;
+ }
+ YT_ABORT();
+}
+
+NQueryTrackerClient::EQueryEngine ConvertQueryEngineFromProto(
+ NProto::EQueryEngine proto)
+{
+ switch (proto) {
+ case NProto::EQueryEngine::QE_QL:
+ return NQueryTrackerClient::EQueryEngine::Ql;
+ case NProto::EQueryEngine::QE_YQL:
+ return NQueryTrackerClient::EQueryEngine::Yql;
+ case NProto::EQueryEngine::QE_CHYT:
+ return NQueryTrackerClient::EQueryEngine::Chyt;
+ case NProto::EQueryEngine::QE_MOCK:
+ return NQueryTrackerClient::EQueryEngine::Mock;
+ case NProto::EQueryEngine::QE_SPYT:
+ return NQueryTrackerClient::EQueryEngine::Spyt;
+ case NProto::EQueryEngine::QE_UNKNOWN:
+ THROW_ERROR_EXCEPTION("Protobuf contains unknown value for query engine");
+ }
+ YT_ABORT();
+}
+
+NProto::EQueryState ConvertQueryStateToProto(
+ NQueryTrackerClient::EQueryState queryState)
+{
+ switch (queryState) {
+ case NQueryTrackerClient::EQueryState::Draft:
+ return NProto::EQueryState::QS_DRAFT;
+ case NQueryTrackerClient::EQueryState::Pending:
+ return NProto::EQueryState::QS_PENDING;
+ case NQueryTrackerClient::EQueryState::Running:
+ return NProto::EQueryState::QS_RUNNING;
+ case NQueryTrackerClient::EQueryState::Aborting:
+ return NProto::EQueryState::QS_ABORTING;
+ case NQueryTrackerClient::EQueryState::Aborted:
+ return NProto::EQueryState::QS_ABORTED;
+ case NQueryTrackerClient::EQueryState::Completing:
+ return NProto::EQueryState::QS_COMPLETING;
+ case NQueryTrackerClient::EQueryState::Completed:
+ return NProto::EQueryState::QS_COMPLETED;
+ case NQueryTrackerClient::EQueryState::Failing:
+ return NProto::EQueryState::QS_FAILING;
+ case NQueryTrackerClient::EQueryState::Failed:
+ return NProto::EQueryState::QS_FAILED;
+ }
+ YT_ABORT();
+}
+
+NQueryTrackerClient::EQueryState ConvertQueryStateFromProto(
+ NProto::EQueryState proto)
+{
+ switch (proto) {
+ case NProto::EQueryState::QS_DRAFT:
+ return NQueryTrackerClient::EQueryState::Draft;
+ case NProto::EQueryState::QS_PENDING:
+ return NQueryTrackerClient::EQueryState::Pending;
+ case NProto::EQueryState::QS_RUNNING:
+ return NQueryTrackerClient::EQueryState::Running;
+ case NProto::EQueryState::QS_ABORTING:
+ return NQueryTrackerClient::EQueryState::Aborting;
+ case NProto::EQueryState::QS_ABORTED:
+ return NQueryTrackerClient::EQueryState::Aborted;
+ case NProto::EQueryState::QS_COMPLETING:
+ return NQueryTrackerClient::EQueryState::Completing;
+ case NProto::EQueryState::QS_COMPLETED:
+ return NQueryTrackerClient::EQueryState::Completed;
+ case NProto::EQueryState::QS_FAILING:
+ return NQueryTrackerClient::EQueryState::Failing;
+ case NProto::EQueryState::QS_FAILED:
+ return NQueryTrackerClient::EQueryState::Failed;
+ case NProto::EQueryState::QS_UNKNOWN:
+ THROW_ERROR_EXCEPTION("Protobuf contains unknown value for query state");
+ }
+ YT_ABORT();
+}
+
////////////////////////////////////////////////////////////////////////////////
} // namespace NProto
diff --git a/yt/yt/client/api/rpc_proxy/helpers.h b/yt/yt/client/api/rpc_proxy/helpers.h
index 4b01c4ae11..722e47380e 100644
--- a/yt/yt/client/api/rpc_proxy/helpers.h
+++ b/yt/yt/client/api/rpc_proxy/helpers.h
@@ -228,6 +228,14 @@ void FromProto(
NApi::TBackupManifest* manifest,
const NProto::TBackupManifest& protoManifest);
+void ToProto(
+ NProto::TQuery* protoQuery,
+ const NApi::TQuery& query);
+
+void FromProto(
+ NApi::TQuery* query,
+ const NProto::TQuery& protoQuery);
+
NProto::EOperationType ConvertOperationTypeToProto(
NScheduler::EOperationType operationType);
@@ -252,6 +260,17 @@ NProto::EJobState ConvertJobStateToProto(
NJobTrackerClient::EJobState ConvertJobStateFromProto(
NProto::EJobState proto);
+NProto::EQueryEngine ConvertQueryEngineToProto(
+ NQueryTrackerClient::EQueryEngine queryEngine);
+
+NQueryTrackerClient::EQueryEngine ConvertQueryEngineFromProto(
+ NProto::EQueryEngine proto);
+
+NProto::EQueryState ConvertQueryStateToProto(
+ NQueryTrackerClient::EQueryState queryState);
+
+NQueryTrackerClient::EQueryState ConvertQueryStateFromProto(
+ NProto::EQueryState proto);
} // namespace NProto
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/client/table_client/config.cpp b/yt/yt/client/table_client/config.cpp
index 87373d5b95..d8629254d5 100644
--- a/yt/yt/client/table_client/config.cpp
+++ b/yt/yt/client/table_client/config.cpp
@@ -277,7 +277,8 @@ void TDictionaryCompressionConfig::Register(TRegistrar registrar)
.Default({
EDictionaryCompressionPolicy::LargeChunkFirst,
EDictionaryCompressionPolicy::FreshChunkFirst,
- });
+ })
+ .ResetOnLoad();
registrar.Parameter("policy_probation_samples_size", &TThis::PolicyProbationSamplesSize)
.GreaterThan(0)
diff --git a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp
index 71457da7aa..1410442a1a 100644
--- a/yt/yt/core/ytree/unittests/yson_struct_ut.cpp
+++ b/yt/yt/core/ytree/unittests/yson_struct_ut.cpp
@@ -602,12 +602,12 @@ TEST(TYsonStructTest, LoadSingleParameter)
auto config = New<TTestConfig>();
config->NullableInt = 10;
- config->LoadParameter("my_string", ConvertToNode("test"), EMergeStrategy::Default);
+ config->LoadParameter("my_string", ConvertToNode("test"));
EXPECT_EQ("test", config->MyString);
EXPECT_EQ(10, config->NullableInt);
}
-TEST(TYsonStructTest, LoadSingleParameterWithMergeStrategy)
+TEST(TYsonStructTest, LoadSingleParameterOverwriteDefaults)
{
auto builder = CreateBuilderFromFactory(GetEphemeralNodeFactory());
builder->BeginTree();
@@ -619,15 +619,9 @@ TEST(TYsonStructTest, LoadSingleParameterWithMergeStrategy)
auto config1 = New<TTestConfig>();
config1->Subconfig->MyBool = true;
- config1->LoadParameter("sub", subConfig, EMergeStrategy::Default);
+ config1->LoadParameter("sub", subConfig);
EXPECT_EQ(100, config1->Subconfig->MyInt);
- EXPECT_TRUE(config1->Subconfig->MyBool); // Subconfig merged by default.
-
- auto config2 = New<TTestConfig>();
- config2->Subconfig->MyBool = true;
- config2->LoadParameter("sub", subConfig, EMergeStrategy::Overwrite);
- EXPECT_EQ(100, config2->Subconfig->MyInt);
- EXPECT_FALSE(config2->Subconfig->MyBool); // Overwrite destroyed previous values.
+ EXPECT_FALSE(config1->Subconfig->MyBool); // Subconfig is overwritten.
}
TEST(TYsonStructTest, ResetSingleParameter)
@@ -993,11 +987,11 @@ TEST(TYsonStructTest, EnumAsKeyToYHash)
};
TString serialized = "{\"value0\"=\"abc\";}";
- ASSERT_EQ(serialized, ConvertToYsonString(original, EYsonFormat::Text).AsStringBuf());
+ EXPECT_EQ(serialized, ConvertToYsonString(original, EYsonFormat::Text).AsStringBuf());
Deserialize(deserialized, ConvertToNode(TYsonString(serialized, EYsonType::Node)));
- ASSERT_EQ(original, deserialized);
+ EXPECT_EQ(original, deserialized);
}
////////////////////////////////////////////////////////////////////////////////
@@ -1998,8 +1992,7 @@ public:
sub.MyInt = 11;
sub.MyString = "x";
return sub;
- })
- .MergeBy(EMergeStrategy::Combine);
+ });
}
};
@@ -2096,5 +2089,150 @@ TEST(TYsonStructTest, CustomSubExternalizedStruct)
////////////////////////////////////////////////////////////////////////////////
+TIntrusivePtr<TSimpleYsonStruct> CreateSimpleYsonStruct(int value)
+{
+ auto result = New<TSimpleYsonStruct>();
+ result->IntValue = value;
+ return result;
+}
+
+class TTestingNestedListWithCustomDefault
+ : public TYsonStruct
+{
+public:
+ std::vector<TIntrusivePtr<TSimpleYsonStruct>> NestedList;
+
+ REGISTER_YSON_STRUCT(TTestingNestedListWithCustomDefault);
+
+ static void Register(TRegistrar registrar)
+ {
+ registrar.Parameter("nested_list_1", &TThis::NestedList)
+ .DefaultCtor([] {
+ return std::vector{CreateSimpleYsonStruct(5)};
+ });
+ }
+};
+
+TEST(TYsonStructTest, NestedListWithCustomDefault)
+{
+ {
+ auto testInput = TYsonString(TStringBuf("{}"));
+ auto deserialized = ConvertTo<TIntrusivePtr<TTestingNestedListWithCustomDefault>>(testInput);
+
+ EXPECT_EQ(deserialized->NestedList.size(), 1u);
+ EXPECT_EQ(deserialized->NestedList[0]->IntValue, 5);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TTestingNestedMapWithCustomDefault
+ : public TYsonStruct
+{
+public:
+ THashMap<TString, TIntrusivePtr<TSimpleYsonStruct>> NestedMap;
+
+ REGISTER_YSON_STRUCT(TTestingNestedMapWithCustomDefault);
+
+ static void Register(TRegistrar registrar)
+ {
+ registrar.Parameter("nested_map", &TThis::NestedMap)
+ .DefaultCtor([] {
+ return THashMap<TString, TIntrusivePtr<TSimpleYsonStruct>>{
+ {"foo", CreateSimpleYsonStruct(42)},
+ {"bar", CreateSimpleYsonStruct(7)},
+ };
+ });
+ }
+};
+
+TEST(TYsonStructTest, NestedMapWithCustomDefault)
+{
+ {
+ auto testInput = TYsonString(TStringBuf("{}"));
+ auto deserialized = ConvertTo<TIntrusivePtr<TTestingNestedMapWithCustomDefault>>(testInput);
+
+ EXPECT_EQ(deserialized->NestedMap.size(), 2u);
+ EXPECT_EQ(deserialized->NestedMap["foo"]->IntValue, 42);
+ EXPECT_EQ(deserialized->NestedMap["bar"]->IntValue, 7);
+
+ auto testNode = BuildYsonNodeFluently()
+ .BeginMap()
+ .Item("nested_map")
+ .BeginMap()
+ .Item("baz")
+ .BeginMap()
+ .Item("int_value").Value(33)
+ .EndMap()
+ .Item("foo")
+ .BeginMap()
+ .Item("int_value").Value(88)
+ .EndMap()
+ .EndMap()
+ .EndMap();
+ Deserialize(deserialized, testNode->AsMap());
+ EXPECT_EQ(deserialized->NestedMap.size(), 3u);
+ EXPECT_EQ(deserialized->NestedMap["baz"]->IntValue, 33);
+ EXPECT_EQ(deserialized->NestedMap["foo"]->IntValue, 88);
+ EXPECT_EQ(deserialized->NestedMap["bar"]->IntValue, 7);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TTestingNestedMapWithCustomDefaultResetOnLoad
+ : public TYsonStruct
+{
+public:
+ THashMap<TString, TIntrusivePtr<TSimpleYsonStruct>> NestedMap;
+
+ REGISTER_YSON_STRUCT(TTestingNestedMapWithCustomDefaultResetOnLoad);
+
+ static void Register(TRegistrar registrar)
+ {
+ registrar.Parameter("nested_map", &TThis::NestedMap)
+ .DefaultCtor([] {
+ return THashMap<TString, TIntrusivePtr<TSimpleYsonStruct>>{
+ {"foo", CreateSimpleYsonStruct(42)},
+ {"bar", CreateSimpleYsonStruct(7)},
+ };
+ })
+ .ResetOnLoad();
+ }
+};
+
+TEST(TYsonStructTest, NestedMapWithCustomDefaultAndResetOnLoad)
+{
+ {
+ auto testInput = TYsonString(TStringBuf("{}"));
+ auto deserialized = ConvertTo<TIntrusivePtr<TTestingNestedMapWithCustomDefaultResetOnLoad>>(testInput);
+
+ EXPECT_EQ(deserialized->NestedMap.size(), 2u);
+ EXPECT_EQ(deserialized->NestedMap["foo"]->IntValue, 42);
+ EXPECT_EQ(deserialized->NestedMap["bar"]->IntValue, 7);
+
+ auto testNode = BuildYsonNodeFluently()
+ .BeginMap()
+ .Item("nested_map")
+ .BeginMap()
+ .Item("baz")
+ .BeginMap()
+ .Item("int_value").Value(33)
+ .EndMap()
+ .Item("foo")
+ .BeginMap()
+ .Item("int_value").Value(88)
+ .EndMap()
+ .EndMap()
+ .EndMap();
+ Deserialize(deserialized, testNode->AsMap());
+ EXPECT_EQ(deserialized->NestedMap.size(), 2u);
+ EXPECT_EQ(deserialized->NestedMap["baz"]->IntValue, 33);
+ EXPECT_EQ(deserialized->NestedMap["foo"]->IntValue, 88);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
} // namespace
} // namespace NYT::NYTree
diff --git a/yt/yt/core/ytree/yson_struct.cpp b/yt/yt/core/ytree/yson_struct.cpp
index 09b98aebf3..0946a1168a 100644
--- a/yt/yt/core/ytree/yson_struct.cpp
+++ b/yt/yt/core/ytree/yson_struct.cpp
@@ -112,9 +112,9 @@ void TYsonStructBase::SaveParameter(const TString& key, IYsonConsumer* consumer)
Meta_->GetParameter(key)->Save(this, consumer);
}
-void TYsonStructBase::LoadParameter(const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy)
+void TYsonStructBase::LoadParameter(const TString& key, const NYTree::INodePtr& node)
{
- Meta_->LoadParameter(this, key, node, mergeStrategy);
+ Meta_->LoadParameter(this, key, node);
}
void TYsonStructBase::ResetParameter(const TString& key)
diff --git a/yt/yt/core/ytree/yson_struct.h b/yt/yt/core/ytree/yson_struct.h
index ea4a488387..fd66023a47 100644
--- a/yt/yt/core/ytree/yson_struct.h
+++ b/yt/yt/core/ytree/yson_struct.h
@@ -90,7 +90,7 @@ public:
// TODO(renadeen): remove this methods.
void SaveParameter(const TString& key, NYson::IYsonConsumer* consumer) const;
- void LoadParameter(const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy);
+ void LoadParameter(const TString& key, const NYTree::INodePtr& node);
void ResetParameter(const TString& key);
std::vector<TString> GetAllParameterAliases(const TString& key) const;
diff --git a/yt/yt/core/ytree/yson_struct_detail-inl.h b/yt/yt/core/ytree/yson_struct_detail-inl.h
index 6785af863d..0ca9f4ab2f 100644
--- a/yt/yt/core/ytree/yson_struct_detail-inl.h
+++ b/yt/yt/core/ytree/yson_struct_detail-inl.h
@@ -41,13 +41,8 @@ void LoadFromNode(
T& parameter,
NYTree::INodePtr node,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> /*recursiveUnrecognizedStrategy*/)
{
- if (mergeStrategy == EMergeStrategy::Overwrite) {
- parameter = T();
- }
-
try {
Deserialize(parameter, node);
} catch (const std::exception& ex) {
@@ -62,27 +57,12 @@ inline void LoadFromNode(
NYTree::INodePtr& parameter,
NYTree::INodePtr node,
const NYPath::TYPath& /*path*/,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> /*recursiveUnrecognizedStrategy*/)
{
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite: {
- parameter = node;
- break;
- }
-
- case EMergeStrategy::Combine: {
- if (!parameter) {
- parameter = node;
- } else {
- parameter = PatchNode(parameter, node);
- }
- break;
- }
-
- default:
- YT_UNIMPLEMENTED();
+ if (!parameter) {
+ parameter = node;
+ } else {
+ parameter = PatchNode(parameter, node);
}
}
@@ -92,10 +72,9 @@ void LoadFromNode(
TIntrusivePtr<T>& parameterValue,
NYTree::INodePtr node,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
- if (!parameterValue || mergeStrategy == EMergeStrategy::Overwrite) {
+ if (!parameterValue) {
parameterValue = New<T>();
}
@@ -103,17 +82,7 @@ void LoadFromNode(
parameterValue->SetUnrecognizedStrategy(*recursiveUnrecognizedStrategy);
}
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite:
- case EMergeStrategy::Combine: {
- parameterValue->Load(node, false, false, path);
- break;
- }
-
- default:
- YT_UNIMPLEMENTED();
- }
+ parameterValue->Load(node, false, false, path);
}
// YsonStructLite or ExternalizedYsonStruct serializer
@@ -122,17 +91,8 @@ void LoadFromNode(
T& parameter,
NYTree::INodePtr node,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> /*recursiveUnrecognizedStrategy*/)
{
- if (mergeStrategy == EMergeStrategy::Overwrite) {
- // NB: We call SetDefaults here instead of plain T()
- // because ExternalizedYsonStruct serializer doesn't
- // own its data therefore defaulting it would drop the
- // reference to the actual object instead of overwriting it.
- parameter.SetDefaults();
- }
-
try {
parameter.Load(node, /*postprocess*/ true, /*setDefaults*/ false);
} catch (const std::exception& ex) {
@@ -147,13 +107,8 @@ void LoadFromNode(
T& parameter,
NYTree::INodePtr node,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> /*recursiveUnrecognizedStrategy*/)
{
- if (mergeStrategy == EMergeStrategy::Overwrite) {
- parameter = T();
- }
-
try {
DeserializeExternalized(parameter, node, /*postprocess*/ true, /*setDefaults*/ false);
} catch (const std::exception& ex) {
@@ -168,37 +123,19 @@ void LoadFromNode(
std::optional<T>& parameter,
NYTree::INodePtr node,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite: {
- if (node->GetType() == NYTree::ENodeType::Entity) {
- parameter = std::nullopt;
- } else {
- T value;
- LoadFromNode(value, node, path, EMergeStrategy::Overwrite, recursiveUnrecognizedStrategy);
- parameter = std::move(value);
- }
- break;
- }
-
- case EMergeStrategy::Combine: {
- if (node->GetType() != NYTree::ENodeType::Entity) {
- if (parameter.has_value()) {
- LoadFromNode(*parameter, node, path, EMergeStrategy::Combine, recursiveUnrecognizedStrategy);
- } else {
- T value;
- LoadFromNode(value, node, path, EMergeStrategy::Overwrite, recursiveUnrecognizedStrategy);
- parameter = std::move(value);
- }
- }
- break;
- }
+ if (node->GetType() == NYTree::ENodeType::Entity) {
+ parameter = std::nullopt;
+ return;
+ }
- default:
- YT_UNIMPLEMENTED();
+ if (parameter.has_value()) {
+ LoadFromNode(*parameter, node, path, recursiveUnrecognizedStrategy);
+ } else {
+ T value;
+ LoadFromNode(value, node, path, recursiveUnrecognizedStrategy);
+ parameter = std::move(value);
}
}
@@ -208,28 +145,18 @@ void LoadFromNode(
std::vector<T...>& parameter,
NYTree::INodePtr node,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite: {
- auto listNode = node->AsList();
- auto size = listNode->GetChildCount();
- parameter.resize(size);
- for (int i = 0; i < size; ++i) {
- LoadFromNode(
- parameter[i],
- listNode->GetChildOrThrow(i),
- path + "/" + NYPath::ToYPathLiteral(i),
- EMergeStrategy::Overwrite,
- recursiveUnrecognizedStrategy);
- }
- break;
- }
-
- default:
- YT_UNIMPLEMENTED();
+ auto listNode = node->AsList();
+ auto size = listNode->GetChildCount();
+ parameter.clear();
+ parameter.reserve(size);
+ for (int i = 0; i < size; ++i) {
+ LoadFromNode(
+ parameter.emplace_back(),
+ listNode->GetChildOrThrow(i),
+ path + "/" + NYPath::ToYPathLiteral(i),
+ recursiveUnrecognizedStrategy);
}
}
@@ -253,43 +180,17 @@ void LoadFromNode(
Map<T...>& parameter,
NYTree::INodePtr node,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite: {
- auto mapNode = node->AsMap();
- parameter.clear();
- for (const auto& [key, child] : mapNode->GetChildren()) {
- M value;
- LoadFromNode(
- value,
- child,
- path + "/" + NYPath::ToYPathLiteral(key),
- EMergeStrategy::Overwrite,
- recursiveUnrecognizedStrategy);
- parameter.emplace(DeserializeMapKey<typename Map<T...>::key_type>(key), std::move(value));
- }
- break;
- }
- case EMergeStrategy::Combine: {
- auto mapNode = node->AsMap();
- for (const auto& [key, child] : mapNode->GetChildren()) {
- M value;
- LoadFromNode(
- value,
- child,
- path + "/" + NYPath::ToYPathLiteral(key),
- EMergeStrategy::Combine,
- recursiveUnrecognizedStrategy);
- parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value);
- }
- break;
- }
-
- default:
- YT_UNIMPLEMENTED();
+ auto mapNode = node->AsMap();
+ for (const auto& [key, child] : mapNode->GetChildren()) {
+ M value;
+ LoadFromNode(
+ value,
+ child,
+ path + "/" + NYPath::ToYPathLiteral(key),
+ recursiveUnrecognizedStrategy);
+ parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value);
}
}
@@ -302,10 +203,9 @@ void LoadFromCursor(
T& parameter,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
- LoadFromNode(parameter, NYson::ExtractTo<NYTree::INodePtr>(cursor), path, mergeStrategy, recursiveUnrecognizedStrategy);
+ LoadFromNode(parameter, NYson::ExtractTo<NYTree::INodePtr>(cursor), path, recursiveUnrecognizedStrategy);
}
////////////////////////////////////////////////////////////////////////////////
@@ -315,7 +215,6 @@ void LoadFromCursor(
TIntrusivePtr<T>& parameterValue,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy);
template <class... T>
@@ -323,7 +222,6 @@ void LoadFromCursor(
std::vector<T...>& parameter,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy);
// std::optional
@@ -332,7 +230,6 @@ void LoadFromCursor(
std::optional<T>& parameter,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy);
template <template <typename...> class Map, class... T, class M = typename Map<T...>::mapped_type>
@@ -340,7 +237,6 @@ void LoadFromCursor(
Map<T...>& parameter,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy);
////////////////////////////////////////////////////////////////////////////////
@@ -351,12 +247,11 @@ inline void LoadFromCursor(
NYTree::INodePtr& parameter,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
try {
auto node = NYson::ExtractTo<INodePtr>(cursor);
- LoadFromNode(parameter, std::move(node), path, mergeStrategy, recursiveUnrecognizedStrategy);
+ LoadFromNode(parameter, std::move(node), path, recursiveUnrecognizedStrategy);
} catch (const std::exception& ex) {
THROW_ERROR_EXCEPTION("Error loading parameter %v", path)
<< ex;
@@ -369,10 +264,9 @@ void LoadFromCursor(
TIntrusivePtr<T>& parameterValue,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
- if (!parameterValue || mergeStrategy == EMergeStrategy::Overwrite) {
+ if (!parameterValue) {
parameterValue = New<T>();
}
@@ -380,17 +274,7 @@ void LoadFromCursor(
parameterValue->SetUnrecognizedStrategy(*recursiveUnrecognizedStrategy);
}
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite:
- case EMergeStrategy::Combine: {
- parameterValue->Load(cursor, false, false, path);
- break;
- }
-
- default:
- YT_UNIMPLEMENTED();
- }
+ parameterValue->Load(cursor, /*postprocess*/ false, /*setDefaults*/ false, path);
}
// std::optional
@@ -399,41 +283,20 @@ void LoadFromCursor(
std::optional<T>& parameter,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
try {
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite: {
- if ((*cursor)->GetType() == NYson::EYsonItemType::EntityValue) {
- parameter = std::nullopt;
- cursor->Next();
- } else {
- T value;
- LoadFromCursor(value, cursor, path, EMergeStrategy::Overwrite, recursiveUnrecognizedStrategy);
- parameter = std::move(value);
- }
- break;
- }
-
- case EMergeStrategy::Combine: {
- if ((*cursor)->GetType() == NYson::EYsonItemType::EntityValue) {
- cursor->Next();
- } else {
- if (parameter.has_value()) {
- LoadFromCursor(*parameter, cursor, path, EMergeStrategy::Combine, recursiveUnrecognizedStrategy);
- } else {
- T value;
- LoadFromCursor(value, cursor, path, EMergeStrategy::Overwrite, recursiveUnrecognizedStrategy);
- parameter = std::move(value);
- }
- }
- break;
+ if ((*cursor)->GetType() == NYson::EYsonItemType::EntityValue) {
+ parameter = std::nullopt;
+ cursor->Next();
+ } else {
+ if (parameter.has_value()) {
+ LoadFromCursor(*parameter, cursor, path, recursiveUnrecognizedStrategy);
+ } else {
+ T value;
+ LoadFromCursor(value, cursor, path, recursiveUnrecognizedStrategy);
+ parameter = std::move(value);
}
-
- default:
- YT_UNIMPLEMENTED();
}
} catch (const std::exception& ex) {
THROW_ERROR_EXCEPTION("Error loading parameter %v", path)
@@ -447,30 +310,19 @@ void LoadFromCursor(
std::vector<T...>& parameter,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
try {
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite: {
- parameter.clear();
- int index = 0;
- cursor->ParseList([&](NYson::TYsonPullParserCursor* cursor) {
- LoadFromCursor(
- parameter.emplace_back(),
- cursor,
- path + "/" + NYPath::ToYPathLiteral(index),
- EMergeStrategy::Overwrite,
- recursiveUnrecognizedStrategy);
- ++index;
- });
- break;
- }
-
- default:
- YT_UNIMPLEMENTED();
- }
+ parameter.clear();
+ int index = 0;
+ cursor->ParseList([&](NYson::TYsonPullParserCursor* cursor) {
+ LoadFromCursor(
+ parameter.emplace_back(),
+ cursor,
+ path + "/" + NYPath::ToYPathLiteral(index),
+ recursiveUnrecognizedStrategy);
+ ++index;
+ });
} catch (const std::exception& ex) {
THROW_ERROR_EXCEPTION("Error loading parameter %v", path)
<< ex;
@@ -483,44 +335,19 @@ void LoadFromCursor(
Map<T...>& parameter,
NYson::TYsonPullParserCursor* cursor,
const NYPath::TYPath& path,
- EMergeStrategy mergeStrategy,
std::optional<EUnrecognizedStrategy> recursiveUnrecognizedStrategy)
{
try {
- auto doParse = [&] (const auto& setterOrEmplacer, EMergeStrategy mergeStrategy) {
- cursor->ParseMap([&] (NYson::TYsonPullParserCursor* cursor) {
- auto key = ExtractTo<TString>(cursor);
- M value;
- LoadFromCursor(
- value,
- cursor,
- path + "/" + NYPath::ToYPathLiteral(key),
- mergeStrategy,
- recursiveUnrecognizedStrategy);
- setterOrEmplacer(key, std::move(value));
- });
- };
-
- switch (mergeStrategy) {
- case EMergeStrategy::Default:
- case EMergeStrategy::Overwrite: {
- parameter.clear();
- auto emplacer = [&] (auto key, M&& value) {
- parameter.emplace(DeserializeMapKey<typename Map<T...>::key_type>(key), std::move(value));
- };
- doParse(emplacer, EMergeStrategy::Overwrite);
- break;
- }
- case EMergeStrategy::Combine: {
- auto setter = [&] (auto key, M&& value) {
- parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value);
- };
- doParse(setter, EMergeStrategy::Combine);
- break;
- }
- default:
- YT_UNIMPLEMENTED();
- }
+ cursor->ParseMap([&] (NYson::TYsonPullParserCursor* cursor) {
+ auto key = ExtractTo<TString>(cursor);
+ M value;
+ LoadFromCursor(
+ value,
+ cursor,
+ path + "/" + NYPath::ToYPathLiteral(key),
+ recursiveUnrecognizedStrategy);
+ parameter[DeserializeMapKey<typename Map<T...>::key_type>(key)] = std::move(value);
+ });
} catch (const std::exception& ex) {
THROW_ERROR_EXCEPTION("Error loading parameter %v", path)
<< ex;
@@ -645,7 +472,58 @@ inline void InvokeForComposites(const Map<T...>* parameter, const F& func)
////////////////////////////////////////////////////////////////////////////////
-} // namespace NYsonStructDetail
+// all
+template <class T>
+inline void ResetOnLoad(T& parameter)
+{
+ parameter = T();
+}
+
+// TYsonStruct
+template <std::derived_from<TYsonStruct> T>
+inline void ResetOnLoad(TIntrusivePtr<T>& parameter)
+{
+ parameter = New<T>();
+}
+
+// TYsonStructLite or TExternalizedYsonStruct Serializer
+template <std::derived_from<TYsonStructLite> T>
+inline void ResetOnLoad(T& parameter)
+{
+ parameter.SetDefaults();
+}
+
+// INodePtr
+template <>
+inline void ResetOnLoad(INodePtr& parameter)
+{
+ parameter.Reset();
+}
+
+// std::optional
+template <class T>
+inline void ResetOnLoad(std::optional<T>& parameter)
+{
+ parameter.reset();
+}
+
+// std::vector
+template <class T>
+inline void ResetOnLoad(std::vector<T>& parameter)
+{
+ parameter.clear();
+}
+
+// any map
+template <template <typename...> class Map, class... T, class M = typename Map<T...>::mapped_type>
+inline void ResetOnLoad(Map<T...>& parameter)
+{
+ parameter.clear();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NPrivate
////////////////////////////////////////////////////////////////////////////////
@@ -679,7 +557,6 @@ template <class TValue>
TYsonStructParameter<TValue>::TYsonStructParameter(TString key, std::unique_ptr<IYsonFieldAccessor<TValue>> fieldAccessor)
: Key_(std::move(key))
, FieldAccessor_(std::move(fieldAccessor))
- , MergeStrategy_(EMergeStrategy::Default)
{ }
template <class TValue>
@@ -689,11 +566,13 @@ void TYsonStructParameter<TValue>::Load(
const TLoadParameterOptions& options)
{
if (node) {
+ if (ResetOnLoad_) {
+ NPrivate::ResetOnLoad(FieldAccessor_->GetValue(self));
+ }
NPrivate::LoadFromNode(
FieldAccessor_->GetValue(self),
std::move(node),
options.Path,
- options.MergeStrategy.value_or(MergeStrategy_),
options.RecursiveUnrecognizedRecursively);
} else if (!Optional_) {
THROW_ERROR_EXCEPTION("Missing required parameter %v",
@@ -702,41 +581,19 @@ void TYsonStructParameter<TValue>::Load(
}
template <class TValue>
-void TYsonStructParameter<TValue>::SafeLoad(
- TYsonStructBase* self,
- NYTree::INodePtr node,
- const TLoadParameterOptions& options,
- const std::function<void()>& validate)
-{
- if (node) {
- TValue oldValue = FieldAccessor_->GetValue(self);
- try {
- NPrivate::LoadFromNode(
- FieldAccessor_->GetValue(self),
- node,
- options.Path,
- options.MergeStrategy.value_or(MergeStrategy_),
- /*recursivelyUnrecognizedStrategy*/ std::nullopt);
- validate();
- } catch (const std::exception ex) {
- FieldAccessor_->GetValue(self) = oldValue;
- throw;
- }
- }
-}
-
-template <class TValue>
void TYsonStructParameter<TValue>::Load(
TYsonStructBase* self,
NYson::TYsonPullParserCursor* cursor,
const TLoadParameterOptions& options)
{
if (cursor) {
+ if (ResetOnLoad_) {
+ NPrivate::ResetOnLoad(FieldAccessor_->GetValue(self));
+ }
NPrivate::LoadFromCursor(
FieldAccessor_->GetValue(self),
cursor,
options.Path,
- options.MergeStrategy.value_or(MergeStrategy_),
options.RecursiveUnrecognizedRecursively);
} else if (!Optional_) {
THROW_ERROR_EXCEPTION("Missing required parameter %v",
@@ -747,18 +604,18 @@ void TYsonStructParameter<TValue>::Load(
template <class TValue>
void TYsonStructParameter<TValue>::SafeLoad(
TYsonStructBase* self,
- NYson::TYsonPullParserCursor* cursor,
+ NYTree::INodePtr node,
const TLoadParameterOptions& options,
const std::function<void()>& validate)
{
- if (cursor) {
+ if (node) {
TValue oldValue = FieldAccessor_->GetValue(self);
try {
- NPrivate::LoadFromCursor(
+ FieldAccessor_->GetValue(self) = TValue();
+ NPrivate::LoadFromNode(
FieldAccessor_->GetValue(self),
- cursor,
+ node,
options.Path,
- options.MergeStrategy.value_or(MergeStrategy_),
/*recursivelyUnrecognizedStrategy*/ std::nullopt);
validate();
} catch (const std::exception ex) {
@@ -839,6 +696,13 @@ TYsonStructParameter<TValue>& TYsonStructParameter<TValue>::Alias(const TString&
}
template <class TValue>
+TYsonStructParameter<TValue>& TYsonStructParameter<TValue>::ResetOnLoad()
+{
+ ResetOnLoad_ = true;
+ return *this;
+}
+
+template <class TValue>
const std::vector<TString>& TYsonStructParameter<TValue>::GetAliases() const
{
return Aliases_;
@@ -921,13 +785,6 @@ TYsonStructParameter<TValue>& TYsonStructParameter<TValue>::CheckThat(TPostproce
}
template <class TValue>
-TYsonStructParameter<TValue>& TYsonStructParameter<TValue>::MergeBy(EMergeStrategy strategy)
-{
- MergeStrategy_ = strategy;
- return *this;
-}
-
-template <class TValue>
IMapNodePtr TYsonStructParameter<TValue>::GetRecursiveUnrecognized(const TYsonStructBase* self) const
{
return NPrivate::TGetRecursiveUnrecognized<TValue>::Do(FieldAccessor_->GetValue(self));
diff --git a/yt/yt/core/ytree/yson_struct_detail.cpp b/yt/yt/core/ytree/yson_struct_detail.cpp
index 48919a5d22..5ccb78dcbc 100644
--- a/yt/yt/core/ytree/yson_struct_detail.cpp
+++ b/yt/yt/core/ytree/yson_struct_detail.cpp
@@ -68,7 +68,7 @@ IYsonStructParameterPtr TYsonStructMeta::GetParameter(const TString& keyOrAlias)
THROW_ERROR_EXCEPTION("Key or alias %Qv not found in yson struct", keyOrAlias);
}
-void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) const
+void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node) const
{
const auto& parameter = GetParameter(key);
auto validate = [&] () {
@@ -87,7 +87,6 @@ void TYsonStructMeta::LoadParameter(TYsonStructBase* target, const TString& key,
};
auto loadOptions = TLoadParameterOptions{
.Path = "",
- .MergeStrategy = mergeStrategy
};
parameter->SafeLoad(target, node, loadOptions, validate);
diff --git a/yt/yt/core/ytree/yson_struct_detail.h b/yt/yt/core/ytree/yson_struct_detail.h
index e6d5a1b595..046a66188d 100644
--- a/yt/yt/core/ytree/yson_struct_detail.h
+++ b/yt/yt/core/ytree/yson_struct_detail.h
@@ -18,7 +18,6 @@ struct TLoadParameterOptions
{
NYPath::TYPath Path;
std::optional<EUnrecognizedStrategy> RecursiveUnrecognizedRecursively;
- std::optional<EMergeStrategy> MergeStrategy;
};
////////////////////////////////////////////////////////////////////////////////
@@ -42,12 +41,6 @@ struct IYsonStructParameter
const TLoadParameterOptions& options,
const std::function<void()>& validate) = 0;
- virtual void SafeLoad(
- TYsonStructBase* self,
- NYson::TYsonPullParserCursor* cursor,
- const TLoadParameterOptions& options,
- const std::function<void()>& validate) = 0;
-
virtual void Save(const TYsonStructBase* self, NYson::IYsonConsumer* consumer) const = 0;
virtual void Postprocess(const TYsonStructBase* self, const NYPath::TYPath& path) const = 0;
@@ -77,7 +70,7 @@ struct IYsonStructMeta
virtual const THashSet<TString>& GetRegisteredKeys() const = 0;
virtual void Postprocess(TYsonStructBase* target, const TYPath& path) const = 0;
virtual IYsonStructParameterPtr GetParameter(const TString& keyOrAlias) const = 0;
- virtual void LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) const = 0;
+ virtual void LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node) const = 0;
virtual void LoadStruct(
TYsonStructBase* target,
@@ -118,7 +111,7 @@ public:
const THashSet<TString>& GetRegisteredKeys() const override;
IYsonStructParameterPtr GetParameter(const TString& keyOrAlias) const override;
- void LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node, EMergeStrategy mergeStrategy) const override;
+ void LoadParameter(TYsonStructBase* target, const TString& key, const NYTree::INodePtr& node) const override;
void Postprocess(TYsonStructBase* target, const TYPath& path) const override;
@@ -223,12 +216,6 @@ public:
NYTree::INodePtr node,
const TLoadParameterOptions& options) override;
- void SafeLoad(
- TYsonStructBase* self,
- NYTree::INodePtr node,
- const TLoadParameterOptions& options,
- const std::function<void()>& validate) override;
-
void Load(
TYsonStructBase* self,
NYson::TYsonPullParserCursor* cursor,
@@ -236,7 +223,7 @@ public:
void SafeLoad(
TYsonStructBase* self,
- NYson::TYsonPullParserCursor* cursor,
+ NYTree::INodePtr node,
const TLoadParameterOptions& options,
const std::function<void()>& validate) override;
@@ -278,8 +265,8 @@ public:
TYsonStructParameter& NonEmpty();
// Register alias for parameter. Used in deserialization.
TYsonStructParameter& Alias(const TString& name);
- // Set merge strategy for parameter
- TYsonStructParameter& MergeBy(EMergeStrategy strategy);
+ // Set field to T() (or suitable analogue) before deserializations.
+ TYsonStructParameter& ResetOnLoad();
// Register constructor with parameters as initializer of default value for ref-counted class.
template <class... TArgs>
@@ -293,9 +280,9 @@ private:
bool SerializeDefault_ = true;
std::vector<TPostprocessor> Postprocessors_;
std::vector<TString> Aliases_;
- EMergeStrategy MergeStrategy_ = EMergeStrategy::Default;
bool TriviallyInitializedIntrusivePtr_ = false;
bool Optional_ = false;
+ bool ResetOnLoad_ = false;
};
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/core/ytree/yson_struct_enum.h b/yt/yt/core/ytree/yson_struct_enum.h
index 47824666fc..a35fba6d0d 100644
--- a/yt/yt/core/ytree/yson_struct_enum.h
+++ b/yt/yt/core/ytree/yson_struct_enum.h
@@ -6,13 +6,7 @@
namespace NYT::NYTree {
-////////////////////////////////////////////////////////////////////////////////
-
-DEFINE_ENUM(EMergeStrategy,
- (Default)
- (Overwrite)
- (Combine)
-);
+///////////////////////////////////////////////////////////////////////////////
DEFINE_ENUM(EUnrecognizedStrategy,
(Drop)
diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto
index f20d71c352..f8977a71a4 100644
--- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto
+++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto
@@ -271,6 +271,38 @@ enum EJobSpecSource
JSS_AUTO = 0xFFFF;
}
+enum EQueryEngine
+{
+ QE_UNKNOWN = 100;
+
+ QE_QL = 0;
+ QE_YQL = 1;
+ QE_CHYT = 2;
+ QE_MOCK = 3;
+ QE_SPYT = 4;
+}
+
+enum EContentType
+{
+ CT_RAW_INLINE_DATA = 0;
+ CT_URL = 1;
+}
+
+enum EQueryState
+{
+ QS_UNKNOWN = 100;
+
+ QS_DRAFT = 0;
+ QS_PENDING = 1;
+ QS_RUNNING = 2;
+ QS_ABORTING = 3;
+ QS_ABORTED = 4;
+ QS_COMPLETING = 5;
+ QS_COMPLETED = 6;
+ QS_FAILING = 7;
+ QS_FAILED = 8;
+}
+
// COMPAT(max42).
// A legacy analog of NYT.NYTree.NProto.TAttributeFilter.
// It is different in that universal filter is encoded via all = true
@@ -3004,3 +3036,174 @@ message TCheckPermissionByAclResult
optional string subject_name = 3;
repeated string missing_subjects = 4;
}
+
+////////////////////////////////////////////////////////////////////////////////
+// Query Tracker
+////////////////////////////////////////////////////////////////////////////////
+
+message TReqStartQuery
+{
+ message TQueryFile
+ {
+ required string name = 1;
+ required string content = 2;
+ required EContentType type = 3;
+ }
+
+ required string query_tracker_stage = 1;
+ required EQueryEngine engine = 2;
+ required string query = 3;
+ optional bytes settings = 4; // YSON
+ optional bool draft = 5;
+ optional bytes annotations = 6; // YSON
+ repeated TQueryFile files = 7;
+ optional string access_control_object = 8;
+}
+
+message TRspStartQuery
+{
+ optional NYT.NProto.TGuid query_id = 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TReqAbortQuery
+{
+ required string query_tracker_stage = 1;
+ required NYT.NProto.TGuid query_id = 2;
+ optional string abort_message = 3;
+}
+
+message TRspAbortQuery
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TReqGetQueryResult
+{
+ required string query_tracker_stage = 1;
+ required NYT.NProto.TGuid query_id = 2;
+ required int64 result_index = 3;
+}
+
+message TRspGetQueryResult
+{
+ required NYT.NProto.TGuid query_id = 1;
+ required int64 result_index = 2;
+ optional NYT.NProto.TError error = 3;
+ optional TTableSchema schema = 4;
+ required NYT.NChunkClient.NProto.TDataStatistics data_statistics = 5;
+ required bool is_truncated = 6;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TReqReadQueryResult
+{
+ message TColumns
+ {
+ repeated string items = 1;
+ }
+
+ required string query_tracker_stage = 1;
+ required NYT.NProto.TGuid query_id = 2;
+ required int64 result_index = 3;
+ optional TColumns columns = 4;
+ optional int64 lower_row_index = 5;
+ optional int64 upper_row_index = 6;
+}
+
+message TRspReadQueryResult
+{
+ required TRowsetDescriptor rowset_descriptor = 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TQuery
+{
+ required NYT.NProto.TGuid id = 1;
+ optional EQueryEngine engine = 2;
+ optional string query = 3;
+ optional bytes files = 4; // YSON
+ optional uint64 start_time = 5; // TInstant
+ optional uint64 finish_time = 6; // TInstant
+ optional bytes settings = 7; // YSON
+ optional string user = 8;
+ optional string access_control_object = 9;
+ optional EQueryState state = 10;
+ optional int64 result_count = 11;
+ optional bytes progress = 12; // YSON
+ optional NYT.NProto.TError error = 13;
+ optional bytes annotations = 14; // YSON
+ optional NYT.NYTree.NProto.TAttributeDictionary other_attributes = 15;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TReqGetQuery
+{
+ required string query_tracker_stage = 1;
+ required NYT.NProto.TGuid query_id = 2;
+ optional NYT.NYTree.NProto.TAttributeFilter attributes = 3;
+ optional uint64 timestamp = 4;
+}
+
+message TRspGetQuery
+{
+ optional TQuery query = 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TReqListQueries
+{
+ required string query_tracker_stage = 1;
+ optional uint64 from_time = 2; // TInstant
+ optional uint64 to_time = 3; // TInstant
+ optional uint64 cursor_time = 4; // TInstant
+ optional EOperationSortDirection cursor_direction = 5 [default = OSD_PAST];
+ optional string user_filter = 6;
+ optional EQueryState state_filter = 7;
+ optional EQueryEngine engine_filter = 8;
+ optional string substr_filter = 9;
+ optional uint64 limit = 10 [default = 100];
+ optional NYT.NYTree.NProto.TAttributeFilter attributes = 11;
+}
+
+message TRspListQueries
+{
+ repeated TQuery queries = 1;
+ required bool incomplete = 2;
+ required uint64 timestamp = 3;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TReqAlterQuery
+{
+ required string query_tracker_stage = 1;
+ required NYT.NProto.TGuid query_id = 2;
+ optional bytes annotations = 3; // YSON
+ optional string access_control_object = 4;
+}
+
+message TRspAlterQuery
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TReqGetQueryTrackerInfo
+{
+ required string query_tracker_stage = 1;
+ optional NYT.NYTree.NProto.TAttributeFilter attributes = 2;
+}
+
+message TRspGetQueryTrackerInfo
+{
+ required string cluster_name = 1;
+ required bytes supported_features = 2; // YSON
+ repeated string access_control_objects = 3;
+}