diff options
author | ivanmorozov <ivanmorozov@yandex-team.com> | 2022-12-13 14:01:31 +0300 |
---|---|---|
committer | ivanmorozov <ivanmorozov@yandex-team.com> | 2022-12-13 14:01:31 +0300 |
commit | bd4b65e566a61abd4255d59a050158cdb061328d (patch) | |
tree | 1ad516822628689c43fdd90cdd76502b43b1d477 | |
parent | 52edb881226ad76880cd2ffea01dbf3845454c7a (diff) | |
download | ydb-bd4b65e566a61abd4255d59a050158cdb061328d.tar.gz |
initialize table schema from ss information
29 files changed, 297 insertions, 102 deletions
diff --git a/ydb/core/tx/tiering/rule/CMakeLists.txt b/ydb/core/tx/tiering/rule/CMakeLists.txt index bff4d48a6d..72a76159e5 100644 --- a/ydb/core/tx/tiering/rule/CMakeLists.txt +++ b/ydb/core/tx/tiering/rule/CMakeLists.txt @@ -14,8 +14,9 @@ target_compile_options(tx-tiering-rule PRIVATE target_link_libraries(tx-tiering-rule PUBLIC contrib-libs-cxxsupp yutil - services-metadata-initializer services-metadata-abstract + services-metadata-common + services-metadata-initializer services-bg_tasks-abstract core-tx-schemeshard ) @@ -25,8 +26,6 @@ target_sources(tx-tiering-rule PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/rule/initializer.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/rule/checker.cpp ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/rule/ss_checker.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/rule/timeout.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/tx/tiering/rule/ss_dialog.cpp ) add_global_library_for(tx-tiering-rule.global tx-tiering-rule) @@ -36,8 +35,9 @@ target_compile_options(tx-tiering-rule.global PRIVATE target_link_libraries(tx-tiering-rule.global PUBLIC contrib-libs-cxxsupp yutil - services-metadata-initializer services-metadata-abstract + services-metadata-common + services-metadata-initializer services-bg_tasks-abstract core-tx-schemeshard ) diff --git a/ydb/core/tx/tiering/rule/manager.cpp b/ydb/core/tx/tiering/rule/manager.cpp index 83b7a29e6c..aa16310215 100644 --- a/ydb/core/tx/tiering/rule/manager.cpp +++ b/ydb/core/tx/tiering/rule/manager.cpp @@ -30,12 +30,4 @@ NMetadata::NModifications::TOperationParsingResult TTieringRulesManager::DoBuild return result; } -NMetadata::NModifications::TTableSchema TTieringRulesManager::ConstructActualSchema() const { - NMetadata::NModifications::TTableSchema result; - result.AddColumn(true, NMetadata::NInternal::TYDBColumn::Bytes(TTieringRule::TDecoder::TieringRuleId)) - .AddColumn(false, NMetadata::NInternal::TYDBColumn::Bytes(TTieringRule::TDecoder::DefaultColumn)) - .AddColumn(false, NMetadata::NInternal::TYDBColumn::Bytes(TTieringRule::TDecoder::Description)); - return result; -} - } diff --git a/ydb/core/tx/tiering/rule/manager.h b/ydb/core/tx/tiering/rule/manager.h index a88cc15b9f..371e931ef0 100644 --- a/ydb/core/tx/tiering/rule/manager.h +++ b/ydb/core/tx/tiering/rule/manager.h @@ -13,8 +13,6 @@ protected: virtual NMetadata::NModifications::TOperationParsingResult DoBuildPatchFromSettings(const NYql::TObjectSettingsImpl& settings, const NMetadata::NModifications::IOperationsManager::TModificationContext& /*context*/) const override; - - virtual NMetadata::NModifications::TTableSchema ConstructActualSchema() const override; }; } diff --git a/ydb/core/tx/tiering/rule/ss_checker.h b/ydb/core/tx/tiering/rule/ss_checker.h index f2932523b4..65a033330d 100644 --- a/ydb/core/tx/tiering/rule/ss_checker.h +++ b/ydb/core/tx/tiering/rule/ss_checker.h @@ -1,12 +1,12 @@ #pragma once #include "object.h" -#include "ss_dialog.h" #include <ydb/core/base/appdata.h> #include <ydb/core/base/tablet_pipecache.h> #include <ydb/core/tx/schemeshard/schemeshard.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/tx/tx_proxy/proxy.h> +#include <ydb/services/metadata/common/ss_dialog.h> namespace NKikimr::NColumnShard::NTiers { @@ -35,9 +35,9 @@ public: } }; -class TSSFetchingActor: public TSSDialogActor { +class TSSFetchingActor: public NMetadata::NInternal::TSSDialogActor { private: - using TBase = TSSDialogActor; + using TBase = NMetadata::NInternal::TSSDialogActor; NSchemeShard::ISSDataProcessor::TPtr Processor; ISSFetchingController::TPtr Controller; void Handle(NSchemeShard::TEvSchemeShard::TEvProcessingResponse::TPtr& ev); diff --git a/ydb/core/tx/tiering/rule/timeout.cpp b/ydb/core/tx/tiering/rule/timeout.cpp deleted file mode 100644 index b1982635cd..0000000000 --- a/ydb/core/tx/tiering/rule/timeout.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "timeout.h" - -namespace NKikimr::NColumnShard::NTiers { - -} diff --git a/ydb/core/tx/tiering/tier/manager.cpp b/ydb/core/tx/tiering/tier/manager.cpp index 1944e18ec3..3939a18663 100644 --- a/ydb/core/tx/tiering/tier/manager.cpp +++ b/ydb/core/tx/tiering/tier/manager.cpp @@ -31,11 +31,4 @@ void TTiersManager::DoPrepareObjectsBeforeModification(std::vector<TTierConfig>& TActivationContext::Register(new TTierPreparationActor(std::move(patchedObjects), controller, context)); } -NMetadata::NModifications::TTableSchema TTiersManager::ConstructActualSchema() const { - NMetadata::NModifications::TTableSchema result; - result.AddColumn(true, NMetadata::NInternal::TYDBColumn::Bytes(TTierConfig::TDecoder::TierName)) - .AddColumn(false, NMetadata::NInternal::TYDBColumn::Bytes(TTierConfig::TDecoder::TierConfig)); - return result; -} - } diff --git a/ydb/core/tx/tiering/tier/manager.h b/ydb/core/tx/tiering/tier/manager.h index 062056613b..32f4481567 100644 --- a/ydb/core/tx/tiering/tier/manager.h +++ b/ydb/core/tx/tiering/tier/manager.h @@ -13,8 +13,6 @@ protected: virtual NMetadata::NModifications::TOperationParsingResult DoBuildPatchFromSettings(const NYql::TObjectSettingsImpl& settings, const NMetadata::NModifications::IOperationsManager::TModificationContext& /*context*/) const override; - - virtual NMetadata::NModifications::TTableSchema ConstructActualSchema() const override; public: }; diff --git a/ydb/services/metadata/CMakeLists.txt b/ydb/services/metadata/CMakeLists.txt index 96609b8fa6..e25aecd950 100644 --- a/ydb/services/metadata/CMakeLists.txt +++ b/ydb/services/metadata/CMakeLists.txt @@ -7,6 +7,7 @@ add_subdirectory(abstract) +add_subdirectory(common) add_subdirectory(ds_table) add_subdirectory(initializer) add_subdirectory(manager) diff --git a/ydb/services/metadata/abstract/common.h b/ydb/services/metadata/abstract/common.h index 7995a7dad3..864f031295 100644 --- a/ydb/services/metadata/abstract/common.h +++ b/ydb/services/metadata/abstract/common.h @@ -13,7 +13,7 @@ namespace NKikimr::NMetadata::NProvider { -enum EEvSubscribe { +enum EEvents { EvRefreshSubscriberData = EventSpaceBegin(TKikimrEvents::ES_METADATA_PROVIDER), EvRefresh, EvEnrichSnapshotResult, @@ -28,10 +28,13 @@ enum EEvSubscribe { EvAlterObjects, EvPrepareManager, EvManagerPrepared, + EvTimeout, + EvTableDescriptionFailed, + EvTableDescriptionSuccess, EvEnd }; -static_assert(EEvSubscribe::EvEnd < EventSpaceEnd(TKikimrEvents::ES_METADATA_PROVIDER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_METADATA_PROVIDER)"); +static_assert(EEvents::EvEnd < EventSpaceEnd(TKikimrEvents::ES_METADATA_PROVIDER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_METADATA_PROVIDER)"); class TEvRefreshSubscriberData: public NActors::TEventLocal<TEvRefreshSubscriberData, EvRefreshSubscriberData> { private: diff --git a/ydb/services/metadata/common/CMakeLists.txt b/ydb/services/metadata/common/CMakeLists.txt new file mode 100644 index 0000000000..49675e7c46 --- /dev/null +++ b/ydb/services/metadata/common/CMakeLists.txt @@ -0,0 +1,25 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(services-metadata-common) +target_compile_options(services-metadata-common PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(services-metadata-common PUBLIC + contrib-libs-cxxsupp + yutil + services-metadata-initializer + services-metadata-abstract + services-bg_tasks-abstract + core-tx-schemeshard +) +target_sources(services-metadata-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/services/metadata/common/timeout.cpp + ${CMAKE_SOURCE_DIR}/ydb/services/metadata/common/ss_dialog.cpp +) diff --git a/ydb/core/tx/tiering/rule/ss_dialog.cpp b/ydb/services/metadata/common/ss_dialog.cpp index ebd79e2641..ebe2851c3e 100644 --- a/ydb/core/tx/tiering/rule/ss_dialog.cpp +++ b/ydb/services/metadata/common/ss_dialog.cpp @@ -5,7 +5,7 @@ #include <ydb/core/protos/services.pb.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> -namespace NKikimr::NColumnShard::NTiers { +namespace NKikimr::NMetadata::NInternal { void TSSDialogActor::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& /*ev*/) { OnFail("cannot delivery message to schemeshard"); diff --git a/ydb/core/tx/tiering/rule/ss_dialog.h b/ydb/services/metadata/common/ss_dialog.h index bb97e1161a..76ee93198f 100644 --- a/ydb/core/tx/tiering/rule/ss_dialog.h +++ b/ydb/services/metadata/common/ss_dialog.h @@ -1,12 +1,11 @@ #pragma once - #include "timeout.h" #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/base/tablet_pipecache.h> -namespace NKikimr::NColumnShard::NTiers { +namespace NKikimr::NMetadata::NInternal { class TSSDialogActor: public TTimeoutActor<TSSDialogActor> { private: diff --git a/ydb/services/metadata/common/timeout.cpp b/ydb/services/metadata/common/timeout.cpp new file mode 100644 index 0000000000..2024c77b6d --- /dev/null +++ b/ydb/services/metadata/common/timeout.cpp @@ -0,0 +1,5 @@ +#include "timeout.h" + +namespace NKikimr::NMetadata::NInternal { + +} diff --git a/ydb/core/tx/tiering/rule/timeout.h b/ydb/services/metadata/common/timeout.h index 437d6fb328..170dda6855 100644 --- a/ydb/core/tx/tiering/rule/timeout.h +++ b/ydb/services/metadata/common/timeout.h @@ -1,14 +1,14 @@ #pragma once #include <ydb/core/base/appdata.h> -#include <ydb/core/tx/tiering/common.h> +#include <ydb/services/metadata/abstract/common.h> #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/events.h> -namespace NKikimr::NColumnShard::NTiers { +namespace NKikimr::NMetadata::NInternal { -class TEvTimeout: public NActors::TEventLocal<TEvTimeout, EEvents::EvTimeout> { +class TEvTimeout: public NActors::TEventLocal<TEvTimeout, NProvider::EEvents::EvTimeout> { }; diff --git a/ydb/services/metadata/ds_table/CMakeLists.txt b/ydb/services/metadata/ds_table/CMakeLists.txt index 350ba78180..fbb641ff7b 100644 --- a/ydb/services/metadata/ds_table/CMakeLists.txt +++ b/ydb/services/metadata/ds_table/CMakeLists.txt @@ -13,6 +13,7 @@ target_link_libraries(services-metadata-ds_table PUBLIC yutil cpp-actors-core ydb-core-base + services-metadata-common core-grpc_services-local_rpc core-grpc_services-base ydb-core-grpc_services @@ -22,6 +23,7 @@ target_link_libraries(services-metadata-ds_table PUBLIC target_sources(services-metadata-ds_table PRIVATE ${CMAKE_SOURCE_DIR}/ydb/services/metadata/ds_table/accessor_refresh.cpp ${CMAKE_SOURCE_DIR}/ydb/services/metadata/ds_table/accessor_subscribe.cpp + ${CMAKE_SOURCE_DIR}/ydb/services/metadata/ds_table/scheme_describe.cpp ${CMAKE_SOURCE_DIR}/ydb/services/metadata/ds_table/service.cpp ${CMAKE_SOURCE_DIR}/ydb/services/metadata/ds_table/config.cpp ) diff --git a/ydb/services/metadata/ds_table/accessor_refresh.h b/ydb/services/metadata/ds_table/accessor_refresh.h index 5aace95ad7..da76f6dd81 100644 --- a/ydb/services/metadata/ds_table/accessor_refresh.h +++ b/ydb/services/metadata/ds_table/accessor_refresh.h @@ -9,11 +9,11 @@ namespace NKikimr::NMetadata::NProvider { class TDSAccessorRefresher; -class TEvRefresh: public NActors::TEventLocal<TEvRefresh, EEvSubscribe::EvRefresh> { +class TEvRefresh: public NActors::TEventLocal<TEvRefresh, EEvents::EvRefresh> { public: }; -class TEvYQLResponse: public NActors::TEventLocal<TEvYQLResponse, EEvSubscribe::EvYQLResponse> { +class TEvYQLResponse: public NActors::TEventLocal<TEvYQLResponse, EEvents::EvYQLResponse> { private: YDB_READONLY_DEF(NRequest::TDialogYQLRequest::TResponse, Response); public: @@ -24,7 +24,7 @@ public: } }; -class TEvEnrichSnapshotResult: public NActors::TEventLocal<TEvEnrichSnapshotResult, EEvSubscribe::EvEnrichSnapshotResult> { +class TEvEnrichSnapshotResult: public NActors::TEventLocal<TEvEnrichSnapshotResult, EEvents::EvEnrichSnapshotResult> { private: YDB_READONLY_DEF(NFetcher::ISnapshot::TPtr, EnrichedSnapshot); public: @@ -34,7 +34,7 @@ public: } }; -class TEvEnrichSnapshotProblem: public NActors::TEventLocal<TEvEnrichSnapshotProblem, EEvSubscribe::EvEnrichSnapshotProblem> { +class TEvEnrichSnapshotProblem: public NActors::TEventLocal<TEvEnrichSnapshotProblem, EEvents::EvEnrichSnapshotProblem> { private: YDB_READONLY_DEF(TString, ErrorText); public: diff --git a/ydb/services/metadata/ds_table/accessor_subscribe.h b/ydb/services/metadata/ds_table/accessor_subscribe.h index 292f6e58f6..809d543631 100644 --- a/ydb/services/metadata/ds_table/accessor_subscribe.h +++ b/ydb/services/metadata/ds_table/accessor_subscribe.h @@ -5,7 +5,7 @@ namespace NKikimr::NMetadata::NProvider { class TDSAccessorNotifier; -class TEvAsk: public NActors::TEventLocal<TEvAsk, EEvSubscribe::EvAskLocal> { +class TEvAsk: public NActors::TEventLocal<TEvAsk, EEvents::EvAskLocal> { private: YDB_READONLY_DEF(TActorId, RequesterId); public: @@ -15,7 +15,7 @@ public: } }; -class TEvSubscribe: public NActors::TEventLocal<TEvSubscribe, EEvSubscribe::EvSubscribeLocal> { +class TEvSubscribe: public NActors::TEventLocal<TEvSubscribe, EEvents::EvSubscribeLocal> { private: YDB_READONLY_DEF(TActorId, SubscriberId); public: @@ -25,7 +25,7 @@ public: } }; -class TEvUnsubscribe: public NActors::TEventLocal<TEvUnsubscribe, EEvSubscribe::EvUnsubscribeLocal> { +class TEvUnsubscribe: public NActors::TEventLocal<TEvUnsubscribe, EEvents::EvUnsubscribeLocal> { private: YDB_READONLY_DEF(TActorId, SubscriberId); public: diff --git a/ydb/services/metadata/ds_table/scheme_describe.cpp b/ydb/services/metadata/ds_table/scheme_describe.cpp new file mode 100644 index 0000000000..baed2ec3e7 --- /dev/null +++ b/ydb/services/metadata/ds_table/scheme_describe.cpp @@ -0,0 +1,62 @@ +#include "scheme_describe.h" + +#include <ydb/core/tx/scheme_cache/scheme_cache.h> +#include <ydb/core/tx/schemeshard/schemeshard.h> +#include <ydb/core/protos/services.pb.h> +#include <ydb/core/ydb_convert/ydb_convert.h> +#include <ydb/core/ydb_convert/table_description.h> + +namespace NKikimr::NMetadata::NProvider { + +void TSchemeDescriptionActor::Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { + auto g = PassAwayGuard(); + const auto& record = ev->Get()->GetRecord(); + const auto status = record.GetStatus(); + if (record.HasReason()) { + auto issue = NYql::TIssue(record.GetReason()); + Controller->OnDescriptionFailed(issue.ToString(), RequestId); + return; + } + Ydb::Table::DescribeTableResult describeTableResult; + if (status != NKikimrScheme::StatusSuccess) { + Controller->OnDescriptionFailed(::ToString(status), RequestId); + return; + } + const auto& pathDescription = record.GetPathDescription(); + Ydb::Scheme::Entry* selfEntry = describeTableResult.mutable_self(); + selfEntry->set_name(pathDescription.GetSelf().GetName()); + selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(pathDescription.GetSelf().GetPathType())); + ConvertDirectoryEntry(pathDescription.GetSelf(), selfEntry, true); + + if (pathDescription.HasColumnTableDescription()) { + const auto& tableDescription = pathDescription.GetColumnTableDescription(); + FillColumnDescription(describeTableResult, tableDescription); + } else { + const auto& tableDescription = pathDescription.GetTable(); + NKikimrMiniKQL::TType splitKeyType; + + try { + FillColumnDescription(describeTableResult, splitKeyType, tableDescription); + } catch (const std::exception& ex) { + Controller->OnDescriptionFailed("Unable to fill column description: " + CurrentExceptionMessage(), RequestId); + return; + } + + describeTableResult.mutable_primary_key()->CopyFrom(tableDescription.GetKeyColumnNames()); + } + Controller->OnDescriptionSuccess(std::move(describeTableResult), RequestId); +} + +void TSchemeDescriptionActor::Execute() { + auto event = std::make_unique<NSchemeShard::TEvSchemeShard::TEvDescribeScheme>(Path); + event->Record.MutableOptions()->SetReturnPartitioningInfo(false); + event->Record.MutableOptions()->SetReturnPartitionConfig(false); + event->Record.MutableOptions()->SetReturnChildren(false); + Send(SchemeShardPipe, new TEvPipeCache::TEvForward(event.release(), SchemeShardId, false)); +} + +void TSchemeDescriptionActor::OnFail(const TString& errorMessage) { + Controller->OnDescriptionFailed(errorMessage, RequestId); +} + +} diff --git a/ydb/services/metadata/ds_table/scheme_describe.h b/ydb/services/metadata/ds_table/scheme_describe.h new file mode 100644 index 0000000000..41f0b76cff --- /dev/null +++ b/ydb/services/metadata/ds_table/scheme_describe.h @@ -0,0 +1,49 @@ +#pragma once + +#include <ydb/core/tx/schemeshard/schemeshard.h> +#include <ydb/services/metadata/common/ss_dialog.h> + +namespace NKikimr::NMetadata::NProvider { + +class ISchemeDescribeController { +public: + using TPtr = std::shared_ptr<ISchemeDescribeController>; + virtual ~ISchemeDescribeController() = default; + virtual void OnDescriptionFailed(const TString& errorMessage, const TString& requestId) const = 0; + virtual void OnDescriptionSuccess(Ydb::Table::DescribeTableResult&& result, const TString& requestId) const = 0; +}; + +class TSchemeDescriptionActor: public NInternal::TSSDialogActor { +private: + using TBase = NInternal::TSSDialogActor; + ISchemeDescribeController::TPtr Controller; + TString Path; + TString RequestId; + void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev); +protected: + virtual void OnBootstrap() override { + Become(&TSchemeDescriptionActor::StateMain); + TBase::OnBootstrap(); + } + virtual void OnFail(const TString& errorMessage) override; + virtual void Execute() override; +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType(); + STFUNC(StateMain) { + switch (ev->GetTypeRewrite()) { + hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, Handle); + default: + TBase::StateMain(ev, ctx); + } + } + TSchemeDescriptionActor(ISchemeDescribeController::TPtr controller, const TString& reqId, const TString& path, const TDuration livetime) + : TBase(livetime) + , Controller(controller) + , Path(path) + , RequestId(reqId) + { + + } +}; + +} diff --git a/ydb/services/metadata/ds_table/service.cpp b/ydb/services/metadata/ds_table/service.cpp index ce4af34759..b4d863e854 100644 --- a/ydb/services/metadata/ds_table/service.cpp +++ b/ydb/services/metadata/ds_table/service.cpp @@ -37,18 +37,17 @@ void TService::PrepareManagers(std::vector<IClassBehaviour::TPtr> managers, TAut void TService::Handle(TEvPrepareManager::TPtr& ev) { auto it = RegisteredManagers.find(ev->Get()->GetManager()->GetTypeId()); if (it != RegisteredManagers.end()) { - Send(ev->Sender, new TEvManagerPrepared(ev->Get()->GetManager())); + Send(ev->Sender, new TEvManagerPrepared(it->second)); } else { auto m = ev->Get()->GetManager(); PrepareManagers({ m }, ev->ReleaseBase(), ev->Sender); } } -void TService::Handle(NInitializer::TEvInitializationFinished::TPtr& ev) { - const TString& initId = ev->Get()->GetInitializationId(); - +void TService::InitializationFinished(const TString& initId) { auto it = ManagersInRegistration.find(initId); Y_VERIFY(it != ManagersInRegistration.end()); + RegisteredManagers.emplace(initId, it->second); ManagersInRegistration.erase(it); @@ -80,6 +79,32 @@ void TService::Handle(NInitializer::TEvInitializationFinished::TPtr& ev) { } } +void TService::Handle(TEvTableDescriptionSuccess::TPtr& ev) { + const TString& initId = ev->Get()->GetRequestId(); + auto it = ManagersInRegistration.find(initId); + Y_VERIFY(it != ManagersInRegistration.end()); + it->second->GetOperationsManager()->SetActualSchema(ev->Get()->GetSchema()); + InitializationFinished(initId); +} + +void TService::Handle(TEvTableDescriptionFailed::TPtr& ev) { + const TString& initId = ev->Get()->GetRequestId(); + ALS_INFO(NKikimrServices::METADATA_PROVIDER) << "metadata service cannot receive table description for " << initId << Endl; + Schedule(TDuration::Seconds(1), new NInitializer::TEvInitializationFinished(initId)); +} + +void TService::Handle(NInitializer::TEvInitializationFinished::TPtr& ev) { + const TString& initId = ev->Get()->GetInitializationId(); + + auto it = ManagersInRegistration.find(initId); + Y_VERIFY(it != ManagersInRegistration.end()); + if (it->second->GetOperationsManager()) { + Register(new TSchemeDescriptionActor(InternalController, initId, it->second->GetStorageTablePath(), TDuration::Seconds(5))); + } else { + InitializationFinished(initId); + } +} + void TService::Handle(TEvSubscribeExternal::TPtr& ev) { const TActorId senderId = ev->Sender; ProcessEventWithFetcher(ev, [this, senderId](const TActorId& actorId) { @@ -97,6 +122,7 @@ void TService::Handle(TEvAskSnapshot::TPtr& ev) { void TService::Handle(TEvObjectsOperation::TPtr& ev) { auto it = RegisteredManagers.find(ev->Get()->GetCommand()->GetManager()->GetTypeId()); if (it != RegisteredManagers.end()) { + ev->Get()->GetCommand()->SetManager(it->second); ev->Get()->GetCommand()->Execute(); } else { auto m = ev->Get()->GetCommand()->GetManager(); @@ -134,4 +160,12 @@ void TServiceInternalController::InitializationFinished(const TString& id) const ActorId.Send(ActorId, new NInitializer::TEvInitializationFinished(id)); } +void TServiceInternalController::OnDescriptionFailed(const TString& errorMessage, const TString& requestId) const { + ActorId.Send(ActorId, new TEvTableDescriptionFailed(errorMessage, requestId)); +} + +void TServiceInternalController::OnDescriptionSuccess(Ydb::Table::DescribeTableResult&& result, const TString& requestId) const { + ActorId.Send(ActorId, new TEvTableDescriptionSuccess(std::move(result), requestId)); +} + } diff --git a/ydb/services/metadata/ds_table/service.h b/ydb/services/metadata/ds_table/service.h index 11907f97b4..4552173cec 100644 --- a/ydb/services/metadata/ds_table/service.h +++ b/ydb/services/metadata/ds_table/service.h @@ -1,6 +1,7 @@ #pragma once #include "accessor_subscribe.h" #include "config.h" +#include "scheme_describe.h" #include <ydb/services/metadata/service.h> #include <ydb/services/metadata/initializer/common.h> @@ -8,12 +9,41 @@ #include <ydb/services/metadata/initializer/manager.h> #include <ydb/services/metadata/initializer/snapshot.h> #include <ydb/services/metadata/initializer/fetcher.h> +#include <ydb/services/metadata/manager/abstract.h> #include <library/cpp/actors/core/hfunc.h> namespace NKikimr::NMetadata::NProvider { -class TServiceInternalController: public NInitializer::IInitializerOutput { +class TEvTableDescriptionFailed: public TEventLocal<TEvTableDescriptionFailed, EEvents::EvTableDescriptionFailed> { +private: + YDB_READONLY_DEF(TString, ErrorMessage); + YDB_READONLY_DEF(TString, RequestId); +public: + explicit TEvTableDescriptionFailed(const TString& errorMessage, const TString& reqId) + : ErrorMessage(errorMessage) + , RequestId(reqId) { + + } +}; + +class TEvTableDescriptionSuccess: public TEventLocal<TEvTableDescriptionSuccess, EEvents::EvTableDescriptionSuccess> { +private: + YDB_READONLY_DEF(TString, RequestId); + YDB_READONLY_DEF(Ydb::Table::DescribeTableResult, Description); +public: + TEvTableDescriptionSuccess(Ydb::Table::DescribeTableResult&& description, const TString& reqId) + : RequestId(reqId) + , Description(std::move(description)) + { + } + + NModifications::TTableSchema GetSchema() const { + return NModifications::TTableSchema(Description); + } +}; + +class TServiceInternalController: public NInitializer::IInitializerOutput, public ISchemeDescribeController { private: const NActors::TActorIdentity ActorId; public: @@ -24,6 +54,9 @@ public: } virtual void InitializationFinished(const TString& id) const override; + + virtual void OnDescriptionFailed(const TString& errorMessage, const TString& requestId) const override; + virtual void OnDescriptionSuccess(Ydb::Table::DescribeTableResult&& result, const TString& requestId) const override; }; class TManagersId { @@ -112,7 +145,12 @@ private: void Handle(TEvSubscribeExternal::TPtr& ev); void Handle(TEvUnsubscribeExternal::TPtr& ev); void Handle(TEvObjectsOperation::TPtr& ev); + void Handle(TEvTableDescriptionSuccess::TPtr& ev); + void Handle(TEvTableDescriptionFailed::TPtr& ev); + void PrepareManagers(std::vector<IClassBehaviour::TPtr> manager, TAutoPtr<IEventBase> ev, const NActors::TActorId& sender); + void InitializationFinished(const TString& initId); + void RequestTableDescription(const TString& path) const; template <class TEventPtr, class TAction> void ProcessEventWithFetcher(TEventPtr& ev, TAction action) { @@ -146,6 +184,10 @@ public: hFunc(TEvPrepareManager, Handle); hFunc(TEvSubscribeExternal, Handle); hFunc(TEvUnsubscribeExternal, Handle); + + hFunc(TEvTableDescriptionSuccess, Handle); + hFunc(TEvTableDescriptionFailed, Handle); + hFunc(NInitializer::TEvInitializationFinished, Handle); default: Y_VERIFY(false); diff --git a/ydb/services/metadata/initializer/manager.cpp b/ydb/services/metadata/initializer/manager.cpp index 0fcdbc0e7d..55859c8b2f 100644 --- a/ydb/services/metadata/initializer/manager.cpp +++ b/ydb/services/metadata/initializer/manager.cpp @@ -17,12 +17,4 @@ NModifications::TOperationParsingResult TManager::DoBuildPatchFromSettings( return result; } -NModifications::TTableSchema TManager::ConstructActualSchema() const { - NModifications::TTableSchema result; - result.AddColumn(true, NInternal::TYDBColumn::Bytes(TDBInitialization::TDecoder::ComponentId)) - .AddColumn(true, NInternal::TYDBColumn::Bytes(TDBInitialization::TDecoder::ModificationId)) - .AddColumn(false, NInternal::TYDBColumn::UInt32(TDBInitialization::TDecoder::Instant)); - return result; -} - } diff --git a/ydb/services/metadata/initializer/manager.h b/ydb/services/metadata/initializer/manager.h index 5e8403d81c..7c004d33ab 100644 --- a/ydb/services/metadata/initializer/manager.h +++ b/ydb/services/metadata/initializer/manager.h @@ -20,7 +20,6 @@ protected: virtual NModifications::TOperationParsingResult DoBuildPatchFromSettings(const NYql::TObjectSettingsImpl& /*settings*/, const TModificationContext& /*context*/) const override; - virtual NModifications::TTableSchema ConstructActualSchema() const override; public: }; diff --git a/ydb/services/metadata/manager/abstract.cpp b/ydb/services/metadata/manager/abstract.cpp index 5af027f1a4..021fe5dc14 100644 --- a/ydb/services/metadata/manager/abstract.cpp +++ b/ydb/services/metadata/manager/abstract.cpp @@ -2,4 +2,37 @@ namespace NKikimr::NMetadata::NModifications { +TTableSchema::TTableSchema(const Ydb::Table::DescribeTableResult& description) { + std::map<TString, Ydb::Column> columns; + for (auto&& i : description.columns()) { + Ydb::Column column; + column.set_name(i.name()); + if (i.type().has_optional_type()) { + *column.mutable_type() = i.type().optional_type().item(); + } else { + *column.mutable_type() = i.type(); + } + columns.emplace(i.name(), std::move(column)); + } + for (auto&& i : description.primary_key()) { + auto it = columns.find(i); + Y_VERIFY(it != columns.end()); + AddColumn(true, it->second); + columns.erase(it); + } + for (auto&& i : columns) { + AddColumn(false, i.second); + } +} + +NKikimr::NMetadata::NModifications::TTableSchema& TTableSchema::AddColumn(const bool primary, const Ydb::Column& info) noexcept { + Columns.emplace_back(primary, info); + YDBColumns.emplace_back(info); + if (primary) { + PKColumns.emplace_back(info); + PKColumnIds.emplace_back(info.name()); + } + return *this; +} + } diff --git a/ydb/services/metadata/manager/abstract.h b/ydb/services/metadata/manager/abstract.h index 558392c726..38a66b39f6 100644 --- a/ydb/services/metadata/manager/abstract.h +++ b/ydb/services/metadata/manager/abstract.h @@ -114,16 +114,10 @@ private: YDB_READONLY_DEF(std::vector<Ydb::Column>, YDBColumns); YDB_READONLY_DEF(std::vector<Ydb::Column>, PKColumns); YDB_READONLY_DEF(std::vector<TString>, PKColumnIds); + + TTableSchema& AddColumn(const bool primary, const Ydb::Column& info) noexcept; public: - TTableSchema& AddColumn(const bool primary, const Ydb::Column& info) noexcept { - Columns.emplace_back(primary, info); - YDBColumns.emplace_back(info); - if (primary) { - PKColumns.emplace_back(info); - PKColumnIds.emplace_back(info.name()); - } - return *this; - } + TTableSchema(const Ydb::Table::DescribeTableResult& description); }; class IOperationsManager { @@ -145,9 +139,7 @@ public: TModificationContext() = default; }; private: - mutable std::optional<TTableSchema> ActualSchema; -protected: - virtual TTableSchema ConstructActualSchema() const = 0; + YDB_ACCESSOR_DEF(std::optional<TTableSchema>, ActualSchema); protected: virtual NThreading::TFuture<TObjectOperatorResult> DoCreateObject(const NYql::TCreateObjectSettings& settings, const ui32 nodeId, IClassBehaviour::TPtr manager, const TModificationContext& context) const = 0; @@ -172,10 +164,9 @@ public: IClassBehaviour::TPtr manager, const TModificationContext& context) const { return DoDropObject(settings, nodeId, manager, context); } + const TTableSchema& GetSchema() const { - if (!ActualSchema) { - ActualSchema = ConstructActualSchema(); - } + Y_VERIFY(!!ActualSchema); return *ActualSchema; } }; @@ -206,7 +197,7 @@ public: class IAlterCommand { private: YDB_READONLY_DEF(std::vector<NInternal::TTableRecord>, Records); - YDB_READONLY_DEF(IClassBehaviour::TPtr, Manager); + YDB_ACCESSOR_DEF(IClassBehaviour::TPtr, Manager); YDB_READONLY_DEF(IAlterController::TPtr, Controller); protected: mutable IOperationsManager::TModificationContext Context; diff --git a/ydb/services/metadata/manager/common.h b/ydb/services/metadata/manager/common.h index 55e2dd5ba7..c2dcbc3ea6 100644 --- a/ydb/services/metadata/manager/common.h +++ b/ydb/services/metadata/manager/common.h @@ -25,7 +25,6 @@ public: virtual void AlterFinished() = 0; }; -} enum EEvents { EvRestoreFinished = EventSpaceBegin(TKikimrEvents::ES_METADATA_MANAGER), @@ -38,6 +37,7 @@ enum EEvents { EvAlterPreparationProblem, EvEnd }; - static_assert(EEvents::EvEnd < EventSpaceEnd(TKikimrEvents::ES_METADATA_MANAGER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_METADATA_MANAGER)"); } + +} diff --git a/ydb/services/metadata/secret/manager.cpp b/ydb/services/metadata/secret/manager.cpp index 1bbe015e26..c1f60a41d0 100644 --- a/ydb/services/metadata/secret/manager.cpp +++ b/ydb/services/metadata/secret/manager.cpp @@ -44,14 +44,6 @@ NModifications::TOperationParsingResult TAccessManager::DoBuildPatchFromSettings return result; } -NModifications::TTableSchema TAccessManager::ConstructActualSchema() const { - NModifications::TTableSchema result; - result.AddColumn(true, NInternal::TYDBColumn::Bytes(TAccess::TDecoder::OwnerUserId)) - .AddColumn(true, NInternal::TYDBColumn::Bytes(TAccess::TDecoder::SecretId)) - .AddColumn(true, NInternal::TYDBColumn::Bytes(TAccess::TDecoder::AccessSID)); - return result; -} - NModifications::TOperationParsingResult TSecretManager::DoBuildPatchFromSettings(const NYql::TObjectSettingsImpl& settings, const NModifications::IOperationsManager::TModificationContext& context) const { NInternal::TTableRecord result; if (!context.GetUserToken()) { @@ -103,12 +95,4 @@ void TSecretManager::DoPrepareObjectsBeforeModification(std::vector<TSecret>&& p TActivationContext::Register(new TSecretPreparationActor(std::move(patchedObjects), controller, context)); } -NModifications::TTableSchema TSecretManager::ConstructActualSchema() const { - NModifications::TTableSchema result; - result.AddColumn(true, NInternal::TYDBColumn::Bytes(TSecret::TDecoder::OwnerUserId)) - .AddColumn(true, NInternal::TYDBColumn::Bytes(TSecret::TDecoder::SecretId)) - .AddColumn(false, NInternal::TYDBColumn::Bytes(TSecret::TDecoder::Value)); - return result; -} - } diff --git a/ydb/services/metadata/secret/manager.h b/ydb/services/metadata/secret/manager.h index 1a135ea866..dd4817ae7e 100644 --- a/ydb/services/metadata/secret/manager.h +++ b/ydb/services/metadata/secret/manager.h @@ -13,7 +13,6 @@ protected: virtual NModifications::TOperationParsingResult DoBuildPatchFromSettings( const NYql::TObjectSettingsImpl& settings, const NModifications::IOperationsManager::TModificationContext& context) const override; - virtual NModifications::TTableSchema ConstructActualSchema() const override; public: }; @@ -26,7 +25,6 @@ protected: virtual NModifications::TOperationParsingResult DoBuildPatchFromSettings(const NYql::TObjectSettingsImpl& settings, const NModifications::IOperationsManager::TModificationContext& context) const override; - virtual NModifications::TTableSchema ConstructActualSchema() const override; public: }; diff --git a/ydb/services/metadata/service.h b/ydb/services/metadata/service.h index f4c1ea5679..a144249101 100644 --- a/ydb/services/metadata/service.h +++ b/ydb/services/metadata/service.h @@ -4,7 +4,7 @@ namespace NKikimr::NMetadata::NProvider { -class TEvObjectsOperation: public NActors::TEventLocal<TEvObjectsOperation, EEvSubscribe::EvAlterObjects> { +class TEvObjectsOperation: public NActors::TEventLocal<TEvObjectsOperation, EEvents::EvAlterObjects> { private: YDB_READONLY_DEF(NModifications::IAlterCommand::TPtr, Command); public: @@ -14,7 +14,7 @@ public: } }; -class TEvPrepareManager: public NActors::TEventLocal<TEvPrepareManager, EEvSubscribe::EvPrepareManager> { +class TEvPrepareManager: public NActors::TEventLocal<TEvPrepareManager, EEvents::EvPrepareManager> { private: YDB_READONLY_DEF(IClassBehaviour::TPtr, Manager); public: @@ -24,7 +24,7 @@ public: } }; -class TEvManagerPrepared: public NActors::TEventLocal<TEvManagerPrepared, EEvSubscribe::EvManagerPrepared> { +class TEvManagerPrepared: public NActors::TEventLocal<TEvManagerPrepared, EEvents::EvManagerPrepared> { private: YDB_READONLY_DEF(IClassBehaviour::TPtr, Manager); public: @@ -34,7 +34,7 @@ public: } }; -class TEvAskSnapshot: public NActors::TEventLocal<TEvAskSnapshot, EEvSubscribe::EvAskExternal> { +class TEvAskSnapshot: public NActors::TEventLocal<TEvAskSnapshot, EEvents::EvAskExternal> { private: YDB_READONLY_DEF(NFetcher::ISnapshotsFetcher::TPtr, Fetcher); public: @@ -44,7 +44,7 @@ public: } }; -class TEvSubscribeExternal: public NActors::TEventLocal<TEvSubscribeExternal, EEvSubscribe::EvSubscribeExternal> { +class TEvSubscribeExternal: public NActors::TEventLocal<TEvSubscribeExternal, EEvents::EvSubscribeExternal> { private: YDB_READONLY_DEF(NFetcher::ISnapshotsFetcher::TPtr, Fetcher); public: @@ -55,7 +55,7 @@ public: } }; -class TEvUnsubscribeExternal: public NActors::TEventLocal<TEvUnsubscribeExternal, EEvSubscribe::EvUnsubscribeExternal> { +class TEvUnsubscribeExternal: public NActors::TEventLocal<TEvUnsubscribeExternal, EEvents::EvUnsubscribeExternal> { private: YDB_READONLY_DEF(NFetcher::ISnapshotsFetcher::TPtr, Fetcher); public: |